Skip to content

Commit d533d5c

Browse files
clouserwclouserw
committed
add wclouser_fxa_db_counts job. Fixes DSRE-1779
1 parent ff7c18b commit d533d5c

File tree

14 files changed

+267
-1
lines changed

14 files changed

+267
-1
lines changed

‎.circleci/config.yml‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ jobs:
139139
name: Test Code
140140
command: docker run app:build python3 -m pytest
141141

142+
142143
build-job-dap-collector-ppa-prod:
143144
docker:
144145
- image: << pipeline.parameters.git-image >>
@@ -155,6 +156,7 @@ jobs:
155156
name: Test Code
156157
command: docker run app:build python3 -m pytest
157158

159+
158160
build-job-desktop-mobile-mau-2020:
159161
docker:
160162
- image: << pipeline.parameters.git-image >>
@@ -345,6 +347,22 @@ jobs:
345347
command: docker run app:build pytest
346348

347349

350+
build-job-wclouser-fxa-db-counts:
351+
docker:
352+
- image: << pipeline.parameters.git-image >>
353+
steps:
354+
- checkout
355+
- compare-branch:
356+
pattern: ^jobs/wclouser-fxa-db-counts/
357+
- setup_remote_docker:
358+
version: << pipeline.parameters.docker-version >>
359+
- run:
360+
name: Build Docker image
361+
command: docker build -t app:build jobs/wclouser-fxa-db-counts/
362+
- run:
363+
name: Test Code
364+
command: docker run app:build pytest --flake8 --black
365+
348366
build-job-webcompat-kb:
349367
docker:
350368
- image: << pipeline.parameters.git-image >>
@@ -501,7 +519,6 @@ workflows:
501519
branches:
502520
only: main
503521

504-
505522
job-fxci-taskcluster-export:
506523
jobs:
507524
- build-job-fxci-taskcluster-export
@@ -620,6 +637,20 @@ workflows:
620637
branches:
621638
only: main
622639

640+
job-wclouser-fxa-db-counts:
641+
jobs:
642+
- build-job-wclouser-fxa-db-counts
643+
- gcp-gcr/build-and-push-image:
644+
context: data-eng-airflow-gcr
645+
docker-context: jobs/wclouser-fxa-db-counts/
646+
path: jobs/wclouser-fxa-db-counts/
647+
image: wclouser-fxa-db-counts_docker_etl
648+
requires:
649+
- build-job-wclouser-fxa-db-counts
650+
filters:
651+
branches:
652+
only: main
653+
623654
job-webcompat-kb:
624655
jobs:
625656
- build-job-webcompat-kb

‎jobs/wclouser-fxa-db-counts/.dockerignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.ci_job.yaml
2+
.ci_workflow.yaml
3+
.DS_Store
4+
*.pyc
5+
.pytest_cache/
6+
__pycache__/
7+
venv/

‎jobs/wclouser-fxa-db-counts/.flake8‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[flake8]
2+
max-line-length = 88

‎jobs/wclouser-fxa-db-counts/.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.DS_Store
2+
*.pyc
3+
__pycache__/
4+
venv/

‎jobs/wclouser-fxa-db-counts/Dockerfile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM python:3.8
2+
MAINTAINER Wil Clouser <[email protected]>
3+
4+
# https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
5+
ARG USER_ID="10001"
6+
ARG GROUP_ID="app"
7+
ARG HOME="/app"
8+
9+
ENV HOME=${HOME}
10+
RUN groupadd --gid ${USER_ID} ${GROUP_ID} && \
11+
useradd --create-home --uid ${USER_ID} --gid ${GROUP_ID} --home-dir ${HOME} ${GROUP_ID}
12+
13+
WORKDIR ${HOME}
14+
15+
RUN pip install --upgrade pip
16+
17+
COPY requirements.txt requirements.txt
18+
RUN pip install -r requirements.txt
19+
20+
COPY . .
21+
22+
RUN pip install .
23+
24+
# Drop root and change ownership of the application folder to the user
25+
RUN chown -R ${USER_ID}:${GROUP_ID} ${HOME}
26+
USER ${USER_ID}

‎jobs/wclouser-fxa-db-counts/README.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# wclouser-fxa-db-counts
2+
3+
This is a simple job to enable us to identify trends within accounts data. E.g. "How many inactive accounts are there?"
4+
5+
## Usage
6+
7+
This script is intended to be run in a docker container.
8+
Build the docker image with:
9+
10+
```sh
11+
docker build -t wclouser_fxa_db_counts .
12+
```
13+
14+
To run locally, install dependencies with:
15+
16+
```sh
17+
pip install -r requirements.txt
18+
```
19+
20+
Run the script with
21+
22+
```sh
23+
python3 -m wclouser_fxa_db_counts.main
24+
```
25+
26+
## Development
27+
28+
Run tests with:
29+
30+
```sh
31+
pytest
32+
```
33+
34+
`flake8` and `black` are included for code linting and formatting:
35+
36+
```sh
37+
pytest --black --flake8
38+
```
39+
40+
or
41+
42+
```sh
43+
flake8 wclouser-fxa-db-counts/ tests/
44+
black --diff wclouser-fxa-db-counts/ tests/
45+
```

‎jobs/wclouser-fxa-db-counts/ci_job.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
build-job-wclouser-fxa-db-counts:
2+
docker:
3+
- image: << pipeline.parameters.git-image >>
4+
steps:
5+
- checkout
6+
- compare-branch:
7+
pattern: ^jobs/wclouser-fxa-db-counts/
8+
- setup_remote_docker:
9+
version: << pipeline.parameters.docker-version >>
10+
- run:
11+
name: Build Docker image
12+
command: docker build -t app:build jobs/wclouser-fxa-db-counts/
13+
- run:
14+
name: Test Code
15+
command: docker run app:build pytest --flake8 --black

‎jobs/wclouser-fxa-db-counts/ci_workflow.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
job-wclouser-fxa-db-counts:
2+
jobs:
3+
- build-job-wclouser-fxa-db-counts
4+
- gcp-gcr/build-and-push-image:
5+
context: data-eng-airflow-gcr
6+
docker-context: jobs/wclouser-fxa-db-counts/
7+
path: jobs/wclouser-fxa-db-counts/
8+
image: wclouser-fxa-db-counts_docker_etl
9+
requires:
10+
- build-job-wclouser-fxa-db-counts
11+
filters:
12+
branches:
13+
only: main

‎jobs/wclouser-fxa-db-counts/pytest.ini‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[pytest]
2+
testpaths =
3+
tests

‎jobs/wclouser-fxa-db-counts/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
click==8.0.4
2+
pytest==6.0.2
3+
pytest-black==0.3.11
4+
pytest-flake8==1.0.6
5+
google-cloud-bigquery==3.26.0

0 commit comments

Comments
 (0)