move build pipeline to circleci (#4923)

.circleci/config.yml

@@ -1,31 +1,73 @@
-version: 2
+version: 2.1
+orbs:
+ aws-cli: circleci/[email protected]
 jobs:
- build:
+ lint:
  docker:
- - image: circleci/python:3.9-buster
+ - image: circleci/python:3.9-bullseye
  steps:
  - checkout
  - run:
  name: Lint with pre-commit
  command: |
  sudo pip install --upgrade pre-commit==2.15.0
  pre-commit run --all-files
+ test-and-build:
+ docker:
+ - image: circleci/python:3.9-bullseye
+ environment:
+ AWS_DEFAULT_REGION: us-west-2
+ steps:
+ - checkout
  - setup_remote_docker:
- version: 17.10.0-ce
+ version: 20.10.7
  docker_layer_caching: true
  - run:
- name: Build docker images
- command: make build-ci
- - run:
- name: Run mocha tests
- command: make test-js-ci
+ name: Build test image
+ command: |
+ cp .env-build .env
+ ./bin/dc_ci.sh build --progress=plain test
  - run:
  # copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
  # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
  name: Copy synonym files to elasticsearch7
  command: |
- ./bin/dc.sh up -d elasticsearch7
+ ./bin/dc_ci.sh up -d elasticsearch7
  docker cp ./kitsune/search/dictionaries/synonyms/. project_elasticsearch7_1:/usr/share/elasticsearch/config/synonyms
  - run:
  name: Run unit tests
- command: make test-ci
+ command: ./bin/dc_ci.sh run test ./bin/run-unit-tests.sh
+ - run:
+ name: Run js tests
+ command: ./bin/dc_ci.sh run test ./bin/run-mocha-tests.sh
+ - when:
+ condition:
+ or:
+ - equal: [ main, << pipeline.git.branch >> ]
+ - equal: [ production, << pipeline.git.branch >> ]
+ - matches: { pattern: "^prod-.+$", value: << pipeline.git.branch >> }
+ steps:
+ - run:
+ name: Build prod image
+ command: ./bin/dc_ci.sh build --progress=plain prod
+ - run:
+ name: Push prod image
+ command: |
+ echo "$DOCKER_PASSWORD" | docker login -u $DOCKER_USERNAME --password-stdin
+ source docker/bin/set_git_env_vars.sh
+ docker image tag mozilla/kitsune:prod-${GIT_COMMIT_SHORT} mozilla/kitsune:prod-latest
+ docker image push mozilla/kitsune:prod-${GIT_COMMIT_SHORT}
+ docker image push mozilla/kitsune:prod-latest
+ docker logout
+ - aws-cli/setup
+ - run:
+ name: Upload staticfiles
+ command: |
+ source docker/bin/set_git_env_vars.sh
+ ./docker/bin/upload-staticfiles.sh
+workflows:
+ version: 2
+ lint-test-build:
+ jobs:
+ - lint
+ - test-and-build

.devcontainer/docker-compose.yml

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
 #-------------------------------------------------------------------------------------------------------------
 
-version: '3.4'
+version: '3.8'
 services:
  # Update this to the name of the service you want to work with in your docker-compose.yml file
  web:

.dockerignore

@@ -16,3 +16,5 @@ __pycache__
 .npm
 .vscode-server
 .cache
+.gnupg*
+styleguide

.env-build

@@ -7,6 +7,5 @@ CELERY_TASK_ALWAYS_EAGER=True
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=sqlite://
 DATABASE_READ_ONLY_URL=sqlite://
-ES7_URLS=elasticsearch:9200
 SESSION_COOKIE_SECURE=False
 SECRET_KEY=secret

.env-test

@@ -1,3 +1,4 @@
+TEST=True
 DEBUG=False
 # Some cron jobs are skipped on stage.
 STAGE=False
@@ -16,7 +17,6 @@ CACHE_URL=redis://redis:6379/3
 CSRF_COOKIE_SECURE=False
 DATABASE_URL=mysql://root:[email protected]:3306/kitsune
 DB_CONN_MAX_AGE=0
-ES_URLS=elasticsearch:9200
 SECRET_KEY=secret
 REUSE_STATIC=1
 REUSE_DB=0

Dockerfile

@@ -1,17 +1,7 @@
-################################
-# Frontend dependencies builder
-#
-FROM node:12 AS frontend-base
-
-WORKDIR /app
-COPY ["./package.json", "./package-lock.json", "prepare_django_assets.js", "/app/"]
-COPY ./kitsune/sumo/static/sumo /app/kitsune/sumo/static/sumo
-RUN npm run production
-
-################################
-# Python dependencies builder
-#
-FROM python:3.9-buster AS base
+#######################
+# Common dependencies #
+#######################
+FROM python:3.9-bullseye AS base
 
 WORKDIR /app
 EXPOSE 8000
@@ -26,83 +16,84 @@ RUN python -m venv /venv
 RUN pip install --upgrade "pip==21.3.1"
 RUN useradd -d /app -M --uid 1000 --shell /usr/sbin/nologin kitsune
 
-RUN apt-get update && \
+RUN apt-get update && apt-get install apt-transport-https && \
+ curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
+ apt-get update && \
  apt-get install -y --no-install-recommends \
  gettext build-essential \
  libxml2-dev libxslt1-dev zlib1g-dev git \
  libjpeg-dev libffi-dev libssl-dev libxslt1.1 \
- libmariadb3 mariadb-client && \
+ libmariadb3 mariadb-client \
+ optipng nodejs zip && \
  rm -rf /var/lib/apt/lists/*
 
 COPY ./requirements/*.txt /app/requirements/
+RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt
 
-RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt && \
- pip install --no-cache-dir --require-hashes -r requirements/dev.txt
 
-ARG GIT_SHA=head
-ENV GIT_SHA=${GIT_SHA}
+#####################
+# Development image #
+#####################
+FROM base AS dev
 
+RUN pip install --no-cache-dir --require-hashes -r requirements/dev.txt
 
-################################
-# Developer image
-#
-FROM base AS base-dev
-RUN apt-get update && apt-get install apt-transport-https && \
- curl -sL https://deb.nodesource.com/setup_12.x | bash -
-RUN apt-get update && apt-get install -y --no-install-recommends optipng nodejs zip && \
- rm -rf /var/lib/apt/lists/*
 
+#########################
+# Frontend dependencies #
+#########################
+FROM base AS base-frontend
 
-################################
-# Fetch locales
-#
-FROM python:3.9-buster AS locales
+COPY package*.json .
+COPY prepare_django_assets.js .
+RUN npm run install-prod && npm run copy:protocol && npm run postinstall
 
-WORKDIR /app
+COPY kitsune/sumo/static/sumo/scss kitsune/sumo/static/sumo/scss
+RUN npm run build:scss && npm run build:postcss
 
-RUN apt-get update && \
- apt-get install -y --no-install-recommends gettext
+COPY . .
+RUN cp .env-build .env && \
+ ./manage.py nunjucks_precompile
 
-ENV PATH="/venv/bin:$PATH"
 
-COPY --from=base /venv /venv
+########################
+# Testing dependencies #
+########################
+FROM base AS test-deps
 
-COPY . .
+RUN pip install --no-cache-dir --require-hashes -r requirements/test.txt
 
-ARG LOCALE_ENV=main
-ENV LOCALE_ENV=${LOCALE_ENV}
-RUN ./docker/bin/fetch-l10n-files.sh
-RUN ./scripts/compile-linted-mo.sh && \
- find ./locale ! -name '*.mo' -type f -delete
 
-ARG GIT_SHA=head
-ENV GIT_SHA ${GIT_SHA}
+#################
+# Testing image #
+#################
+FROM base-frontend AS test
 
+COPY --from=test-deps /venv /venv
 
-################################
-# Staticfiles builder
-#
-FROM base-dev AS staticfiles
+RUN cp .env-test .env && \
+ ./manage.py compilejsi18n && \
+ ./manage.py collectstatic --noinput
 
-COPY --from=frontend-base --chown=kitsune:kitsune /app/assets /app/assets
-COPY --from=frontend-base --chown=kitsune:kitsune /app/node_modules /app/node_modules
-COPY --from=locales /app/locale /app/locale
 
-COPY . .
+##########################
+# Production dependences #
+##########################
+FROM base-frontend AS prod-deps
 
-RUN cp .env-build .env && \
- ./manage.py nunjucks_precompile && \
+RUN ./scripts/l10n-fetch-lint-compile.sh && \
+ find ./locale ! -name '*.mo' -type f -delete && \
  ./manage.py compilejsi18n && \
  # minify jsi18n files:
  find jsi18n/ -name "*.js" -exec sh -c 'npx uglifyjs "$1" -o "${1%.js}-min.js"' sh {} \; && \
  ./manage.py collectstatic --noinput && \
  npx svgo -r -f static
 
 
-################################
-# Full prod image sans locales
-#
-FROM python:3.9-slim-buster AS full-no-locales
+##########################
+# Clean production image #
+##########################
+FROM python:3.9-slim-bullseye AS prod
 
 WORKDIR /app
 
@@ -113,32 +104,25 @@ ENV LANG=C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PYTHONUNBUFFERED=1
 
+RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
+
+COPY --from=prod-deps --chown=kitsune:kitsune /venv /venv
+COPY --from=prod-deps --chown=kitsune:kitsune /app/locale /app/locale
+COPY --from=prod-deps --chown=kitsune:kitsune /app/static /app/static
+
+COPY --chown=kitsune:kitsune . .
+
+# apt-get after copying everything to ensure we're always getting the latest packages in the prod image
 RUN apt-get update && \
+ apt-get upgrade -y && \
  apt-get install -y --no-install-recommends \
  libmariadb3 optipng mariadb-client \
  libxslt1.1 && \
  rm -rf /var/lib/apt/lists/*
 
-RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
-
-COPY --from=base --chown=kitsune:kitsune /venv /venv
-COPY --from=staticfiles --chown=kitsune:kitsune /app/static /app/static
-
-COPY --chown=kitsune:kitsune . .
-
 RUN mkdir /app/media && chown kitsune:kitsune /app/media
 
 USER kitsune
 
 ARG GIT_SHA=head
 ENV GIT_SHA ${GIT_SHA}
-
-
-################################
-# Full final prod image
-#
-FROM full-no-locales AS full
-
-USER root
-COPY --from=locales --chown=kitsune:kitsune /app/locale /app/locale
-USER kitsune