From 8f3d049c931c7dccebf1d27c994609c16828e1f1 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 8 Nov 2024 11:25:21 -0600 Subject: [PATCH 001/156] ci(flake8): run flake8 in gh CI --- .github/workflows/flake8.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/flake8.yml diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml new file mode 100644 index 00000000..d6575f8d --- /dev/null +++ b/.github/workflows/flake8.yml @@ -0,0 +1,30 @@ +name: Run flake8 + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + pull_request: + +jobs: + flake8: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache Docker images. + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }} + + - name: Install dependencies + run: | + pip install flake8 + + - name: Run flake8 + run: | + flake8 From a281350b1c636f7f1e9a475156c173a32588a50c Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 21:51:20 -0600 Subject: [PATCH 002/156] ci(gh-actions): add sast, code quality, and dependency scanning in gh actions start pointing at develop as our default branch to match reality --- .github/workflows/code-quality.yml | 26 +++++++++++++++++++++++ .github/workflows/dependency-scanning.yml | 20 +++++++++++++++++ .github/workflows/docs.yml | 2 +- .github/workflows/flake8.yml | 2 +- .github/workflows/sast.yml | 20 +++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/code-quality.yml create mode 100644 .github/workflows/dependency-scanning.yml create mode 100644 .github/workflows/sast.yml diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 00000000..6be72d85 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,26 @@ +name: Run code quality scans + +on: + # Runs on pushes targeting the default branch + push: + branches: [ "develop" ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + pull_request: + +code-quality: + name: Code Quality + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Run Code Quality Report + run: code-quality-tool --output html + + - name: Upload Code Quality Report + uses: actions/upload-artifact@v3 + with: + name: code-quality-report + path: gl-code-quality-report.html diff --git a/.github/workflows/dependency-scanning.yml b/.github/workflows/dependency-scanning.yml new file mode 100644 index 00000000..b78dc58c --- /dev/null +++ b/.github/workflows/dependency-scanning.yml @@ -0,0 +1,20 @@ +name: Run dependency scanning + +on: + # Runs on pushes targeting the default branch + push: + branches: ["develop"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + pull_request: + +dependency-scanning: + name: Dependency Scanning + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Run Dependency Scanning + run: gemnasium-scan --requirement-file requirements/base.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1e875dc4..7278834a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,7 @@ name: Build sphinx docs on: # Runs on pushes targeting the default branch push: - branches: ["main"] + branches: ["develop"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index d6575f8d..5df2ba55 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -3,7 +3,7 @@ name: Run flake8 on: # Runs on pushes targeting the default branch push: - branches: ["main"] + branches: ["develop"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml new file mode 100644 index 00000000..b5f327de --- /dev/null +++ b/.github/workflows/sast.yml @@ -0,0 +1,20 @@ +name: Run SAST + +on: + # Runs on pushes targeting the default branch + push: + branches: [ "develop" ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + pull_request: + +sast: + name: Static Application Security Testing (SAST) + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Run SAST + run: sast-tool From f6b73f710e47d78bf7f3901ead8bf7ec056db3d5 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 21:58:38 -0600 Subject: [PATCH 003/156] ci(sast): point to an actual sast tool --- .github/workflows/code-quality.yml | 26 ----------------------- .github/workflows/dependency-scanning.yml | 20 ----------------- .github/workflows/sast.yml | 9 ++++++-- 3 files changed, 7 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/dependency-scanning.yml diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 6be72d85..00000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Run code quality scans - -on: - # Runs on pushes targeting the default branch - push: - branches: [ "develop" ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - pull_request: - -code-quality: - name: Code Quality - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Run Code Quality Report - run: code-quality-tool --output html - - - name: Upload Code Quality Report - uses: actions/upload-artifact@v3 - with: - name: code-quality-report - path: gl-code-quality-report.html diff --git a/.github/workflows/dependency-scanning.yml b/.github/workflows/dependency-scanning.yml deleted file mode 100644 index b78dc58c..00000000 --- a/.github/workflows/dependency-scanning.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Run dependency scanning - -on: - # Runs on pushes targeting the default branch - push: - branches: ["develop"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - pull_request: - -dependency-scanning: - name: Dependency Scanning - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Run Dependency Scanning - run: gemnasium-scan --requirement-file requirements/base.txt diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index b5f327de..bc2ff8e0 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -16,5 +16,10 @@ sast: - name: Check out code uses: actions/checkout@v3 - - name: Run SAST - run: sast-tool + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: python + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 From 736ba6b44e4619559379f67942f1bf92eb7cb3cc Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:08:41 -0600 Subject: [PATCH 004/156] ci(pytest): add pytest github action --- .github/workflows/pytest.yml | 70 ++++++++++++++++++++++++++++++++++++ .github/workflows/sast.yml | 25 ++++++------- 2 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 00000000..2a08de3c --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,70 @@ +name: Run pytest and coverage + +on: + # Runs on pushes targeting the default branch + push: + branches: ["develop"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + pull_request: + +jobs: + pytest: + name: Run Pytest + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: scram + POSTGRES_PASSWORD: '' + POSTGRES_DB: test_scram + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U scram" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out the code + uses: actions/checkout@v3 + + - name: Set up Docker + uses: docker/setup-buildx-action@v2 + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose make + + - name: Build Docker images + run: make build + + - name: Migrate Database + run: make migrate + + - name: Run Application + run: make run + + - name: Run Pytest with Coverage + env: + POSTGRES_USER: scram + POSTGRES_DB: test_scram + run: make coverage.xml + + - name: Upload Coverage Report + uses: actions/upload-artifact@v3 + with: + name: coverage-report + path: coverage.xml + + - name: Stop Services + if: always() + run: make stop + + - name: Clean Up + if: always() + run: make clean diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index bc2ff8e0..fba13c77 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -9,17 +9,18 @@ on: workflow_dispatch: pull_request: -sast: - name: Static Application Security Testing (SAST) - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 +jobs: + sast: + name: Static Application Security Testing (SAST) + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: python + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: python - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 From c2e10a54ed47826ce4c8a9e7519e7ab8e1a40c93 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:24:23 -0600 Subject: [PATCH 005/156] ci(sast): test this on pushes in this branch --- .github/workflows/sast.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml index fba13c77..b9417f77 100644 --- a/.github/workflows/sast.yml +++ b/.github/workflows/sast.yml @@ -1,13 +1,18 @@ name: Run SAST on: - # Runs on pushes targeting the default branch push: - branches: [ "develop" ] + branches: + - main + - develop + - topic/soehlert/github_ci # Add your branch here + pull_request: + branches: + - main + - develop # Allows you to run this workflow manually from the Actions tab workflow_dispatch: - pull_request: jobs: sast: From ecaee82ba0d663cfb4c717504bff633bfd4511f8 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:32:00 -0600 Subject: [PATCH 006/156] ci(actions): turn off duplicate SAST and turn on pytest in this branch SAST is already running via the settings on this repo --- .github/workflows/pytest.yml | 13 +++++++++---- .github/workflows/sast.yml | 31 ------------------------------- 2 files changed, 9 insertions(+), 35 deletions(-) delete mode 100644 .github/workflows/sast.yml diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2a08de3c..8bb8625f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,13 +1,18 @@ -name: Run pytest and coverage +name: Run pytest on: - # Runs on pushes targeting the default branch push: - branches: ["develop"] + branches: + - main + - develop + - topic/soehlert/github_ci + pull_request: + branches: + - main + - develop # Allows you to run this workflow manually from the Actions tab workflow_dispatch: - pull_request: jobs: pytest: diff --git a/.github/workflows/sast.yml b/.github/workflows/sast.yml deleted file mode 100644 index b9417f77..00000000 --- a/.github/workflows/sast.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Run SAST - -on: - push: - branches: - - main - - develop - - topic/soehlert/github_ci # Add your branch here - pull_request: - branches: - - main - - develop - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - sast: - name: Static Application Security Testing (SAST) - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: python - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 From 9171cb0f9053b5bed848d4e331a166af63592de6 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:34:51 -0600 Subject: [PATCH 007/156] ci(postgres): make postgres start --- .github/workflows/pytest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 8bb8625f..7e1b47dc 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -26,6 +26,7 @@ jobs: POSTGRES_USER: scram POSTGRES_PASSWORD: '' POSTGRES_DB: test_scram + POSTGRES_HOST_AUTH_METHOD: trust ports: - 5432:5432 options: >- From ab44630eeaa54432a3c1506bf574cd22b70c2521 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:44:49 -0600 Subject: [PATCH 008/156] ci(updates): use latest versions of actions --- .github/workflows/pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7e1b47dc..081a1c34 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -37,10 +37,10 @@ jobs: steps: - name: Check out the code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Install Docker Compose run: | sudo apt-get update @@ -62,7 +62,7 @@ jobs: run: make coverage.xml - name: Upload Coverage Report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: coverage-report path: coverage.xml From 649e016fd5834059dbca7eef3435fb35a719742d Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:55:08 -0600 Subject: [PATCH 009/156] ci(syntax): make all workflows have similar syntax also turn off our testing branch push runs --- .github/workflows/docs.yml | 8 +++++++- .github/workflows/flake8.yml | 9 +++++++-- .github/workflows/pytest.yml | 1 - 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7278834a..3b94bed6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,13 @@ name: Build sphinx docs on: # Runs on pushes targeting the default branch push: - branches: ["develop"] + branches: + - main + - develop + pull_request: + branches: + - main + - develop # Allows you to run this workflow manually from the Actions tab workflow_dispatch: diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 5df2ba55..62041067 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -3,11 +3,16 @@ name: Run flake8 on: # Runs on pushes targeting the default branch push: - branches: ["develop"] + branches: + - main + - develop + pull_request: + branches: + - main + - develop # Allows you to run this workflow manually from the Actions tab workflow_dispatch: - pull_request: jobs: flake8: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 081a1c34..b19f782f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -5,7 +5,6 @@ on: branches: - main - develop - - topic/soehlert/github_ci pull_request: branches: - main From c452612aa796dedbea2e74d32be8482dcbcf1665 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 11 Nov 2024 22:57:20 -0600 Subject: [PATCH 010/156] ci(github-actions): no need for our gitlab CI file anymore --- .gitlab-ci.yml | 68 -------------------------------------------------- 1 file changed, 68 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 435a4812..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,68 +0,0 @@ -include: -- template: Security/Dependency-Scanning.gitlab-ci.yml -- template: Security/Secret-Detection.gitlab-ci.yml -- template: Code-Quality.gitlab-ci.yml -- template: Security/SAST.gitlab-ci.yml - -stages: -- lint -- test - -variables: - POSTGRES_USER: scram - POSTGRES_PASSWORD: '' - POSTGRES_DB: test_scram - POSTGRES_HOST_AUTH_METHOD: trust - -flake8: - stage: lint - image: python:3.8-alpine - before_script: - - pip install -q flake8 - script: - - flake8 - -pytest: - stage: test - image: docker:24.0.6-dind - services: - - docker:dind - variables: - POSTGRES_ENABLED: 1 - before_script: - - apk add make - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - export COMPOSE_PROJECT_NAME=$CI_PIPELINE_ID - - ".ci-scripts/pull_images.sh" - - make build - - ".ci-scripts/push_images.sh" - - make migrate - - make run - script: - - export COMPOSE_PROJECT_NAME=$CI_PIPELINE_ID - - make coverage.xml - artifacts: - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - after_script: - - export COMPOSE_PROJECT_NAME=$CI_PIPELINE_ID - - make stop - - make clean - - -gemnasium-dependency_scanning: - variables: - PIP_REQUIREMENTS_FILE: requirements/base.txt - -code_quality_html: - extends: code_quality - variables: - REPORT_FORMAT: html - artifacts: - paths: - - gl-code-quality-report.html - -sast: - stage: test From 685ca3ba1ac1c4c093569240049add50bc86ff93 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 12 Nov 2024 21:11:59 -0600 Subject: [PATCH 011/156] docs(actions): only run docs action on main --- .github/workflows/docs.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3b94bed6..990b3eaf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,11 +5,6 @@ on: push: branches: - main - - develop - pull_request: - branches: - - main - - develop # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 4bcf56bc5849691e97f32f740c91a1d2272dca0a Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Wed, 13 Nov 2024 21:19:55 -0600 Subject: [PATCH 012/156] ci(run-always): set action to run on every branch works in conjunction with the if on the last step to run always, excpet only publish the docs on the main branch --- .github/workflows/docs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 990b3eaf..1950173c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,7 +4,7 @@ on: # Runs on pushes targeting the default branch push: branches: - - main + - '**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -49,5 +49,7 @@ jobs: path: 'docs/_build/html' - name: Deploy to GitHub Pages + if: github.ref == 'refs/heads/main' id: deployment uses: actions/deploy-pages@v4 + From ce9fd2349cf6185dd575b9d45ab614b2c8c6c3da Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Wed, 13 Nov 2024 21:27:09 -0600 Subject: [PATCH 013/156] ci(run-always): run pytest and flake8 on all pushes we can always skip using a flag in our commit messages if needed https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/skipping-workflow-runs --- .github/workflows/docs.yml | 1 - .github/workflows/flake8.yml | 4 +--- .github/workflows/pytest.yml | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1950173c..1f7e14f0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,7 +1,6 @@ name: Build sphinx docs on: - # Runs on pushes targeting the default branch push: branches: - '**' diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 62041067..7f8d5c53 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -1,11 +1,9 @@ name: Run flake8 on: - # Runs on pushes targeting the default branch push: branches: - - main - - develop + - '**' pull_request: branches: - main diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b19f782f..39ad2089 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,8 +3,7 @@ name: Run pytest on: push: branches: - - main - - develop + - '**' pull_request: branches: - main From 5db525fa83ecf3272cf4fa8f544d0427c651b6dd Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Wed, 13 Nov 2024 21:31:07 -0600 Subject: [PATCH 014/156] ci(coverage): automatically add coverage report to our pull requests --- .github/workflows/pytest.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 39ad2089..0073cd2e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -65,6 +65,11 @@ jobs: name: coverage-report path: coverage.xml + - name: Display Coverage Metrics + uses: 5monkeys/cobertura-action@v14 + with: + minimum_coverage: '50' + - name: Stop Services if: always() run: make stop From 81152c3443d1e4e455f3cb6ad9d8d4f4904bb03b Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 10:29:24 -0600 Subject: [PATCH 015/156] ci(coverage): update coverage configs to focus the MR output --- .github/workflows/pytest.yml | 1 + pyproject.toml | 4 +++- setup.cfg | 6 ------ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0073cd2e..5930abfb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -69,6 +69,7 @@ jobs: uses: 5monkeys/cobertura-action@v14 with: minimum_coverage: '50' + only_changed_files: 'true' - name: Stop Services if: always() diff --git a/pyproject.toml b/pyproject.toml index 54476b25..f5e74ec1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,10 +9,12 @@ python_files = [ # ==== Coverage ==== [tool.coverage.run] -omit = ["*/migrations/*", "*/tests/*"] +include = ["scram/*", "config/*", "translator/*"] +omit = ["**/migrations/*", "scram/contrib/*", "*/tests/*"] plugins = ["django_coverage_plugin"] + # ==== black ==== [tool.black] line-length = 119 diff --git a/setup.cfg b/setup.cfg index 84ec37bb..2c082581 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,12 +24,6 @@ django_settings_module = config.settings.test # Django migrations should not produce any errors: ignore_errors = True -[coverage:run] -include = scram/* -omit = *migrations*, *tests* -plugins = - django_coverage_plugin - [behave] paths = scram/route_manager/tests/acceptance stderr_capture = no From 98a907a418a303d8324de250467c0080889797be Mon Sep 17 00:00:00 2001 From: chriscummings Date: Fri, 8 Nov 2024 13:38:59 -0600 Subject: [PATCH 016/156] chore: bump to django 4.2 --- requirements/base.txt | 12 ++++++------ scram/users/tests/test_forms.py | 2 +- scram/users/tests/test_views.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index cd489161..c378cfc5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,21 +1,21 @@ pytz==2022.1 # https://github.com/stub42/pytz -django-netfields==1.2.2 # https://pypi.org/project/django-netfields/ +django-netfields # https://pypi.org/project/django-netfields/ python-slugify==6.1.2 # https://github.com/un33k/python-slugify Pillow==9.1.1 # https://github.com/python-pillow/Pillow argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi whitenoise==6.2.0 # https://github.com/evansd/whitenoise celery==5.2.7 # pyup: < 6.0 # https://github.com/celery/celery -django-celery-beat==2.3.0 # https://github.com/celery/django-celery-beat +django-celery-beat # https://github.com/celery/django-celery-beat flower==1.0.0 # https://github.com/mher/flower uvicorn[standard]==0.17.6 # https://github.com/encode/uvicorn -asgiref~=3.5.0 # https://github.com/django/asgiref +# asgiref~=3.5.0 # https://github.com/django/asgiref # Django # ------------------------------------------------------------------------------ channels==3.0.4 channels_redis==3.4.0 django-redis==5.3.0 -django==3.2.13 # pyup: < 4.0 # https://www.djangoproject.com/ +django~=4.2 # https://www.djangoproject.com/ django-grip==3.4.0 django-eventstream==4.5.1 # https://github.com/fanout/django-eventstream django-environ==0.9.0 # https://github.com/joke2k/django-environ @@ -25,10 +25,10 @@ django-crispy-forms==1.14.0 # https://github.com/django-crispy-forms/django-cri crispy-bootstrap5==0.6 # https://github.com/django-crispy-forms/crispy-bootstrap5 # Django REST Framework -djangorestframework==3.13.1 # https://github.com/encode/django-rest-framework +# djangorestframework==3.13.1 # https://github.com/encode/django-rest-framework django-cors-headers==3.13.0 # https://github.com/adamchainz/django-cors-headers # DRF-spectacular for api documentation -drf-spectacular==0.22.1 # https://github.com/tfranzel/drf-spectacular +drf-spectacular # https://github.com/tfranzel/drf-spectacular # OIDC # ------------------------------------------------------------------------------ diff --git a/scram/users/tests/test_forms.py b/scram/users/tests/test_forms.py index 8041e621..6d0b248b 100644 --- a/scram/users/tests/test_forms.py +++ b/scram/users/tests/test_forms.py @@ -2,7 +2,7 @@ Module for all Form Tests. """ import pytest -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from scram.users.forms import UserCreationForm from scram.users.models import User diff --git a/scram/users/tests/test_views.py b/scram/users/tests/test_views.py index 708f35e5..485370d2 100644 --- a/scram/users/tests/test_views.py +++ b/scram/users/tests/test_views.py @@ -47,8 +47,8 @@ def test_form_valid(self, user: User, rf: RequestFactory): request = rf.get("/fake-url/") # Add the session/message middleware to the request - SessionMiddleware().process_request(request) - MessageMiddleware().process_request(request) + SessionMiddleware(get_response=request).process_request(request) + MessageMiddleware(get_response=request).process_request(request) request.user = user view.request = request From 75190885f996e01bee79f457d77b7e1361124881 Mon Sep 17 00:00:00 2001 From: chriscummings Date: Thu, 14 Nov 2024 10:40:29 -0600 Subject: [PATCH 017/156] chore(deps): bump python image --- compose/local/django/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index ca5e2cc7..64bd4a6b 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-buster +FROM python:3.12-slim-bookworm ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 From 413e2dc0a8c1c613bf315ae92f08ebf03000d321 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 10:46:08 -0600 Subject: [PATCH 018/156] fix(coverage): only displaying changed files breaks this action --- .github/workflows/pytest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5930abfb..0073cd2e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -69,7 +69,6 @@ jobs: uses: 5monkeys/cobertura-action@v14 with: minimum_coverage: '50' - only_changed_files: 'true' - name: Stop Services if: always() From 6c665543c701bc9754077a930239cd1521d38ded Mon Sep 17 00:00:00 2001 From: chriscummings Date: Thu, 14 Nov 2024 11:09:55 -0600 Subject: [PATCH 019/156] chore(deps): remove pillow pin (it's included by other things) --- requirements/base.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index c378cfc5..50c1a84a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,6 @@ pytz==2022.1 # https://github.com/stub42/pytz django-netfields # https://pypi.org/project/django-netfields/ python-slugify==6.1.2 # https://github.com/un33k/python-slugify -Pillow==9.1.1 # https://github.com/python-pillow/Pillow argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi whitenoise==6.2.0 # https://github.com/evansd/whitenoise celery==5.2.7 # pyup: < 6.0 # https://github.com/celery/celery @@ -25,7 +24,7 @@ django-crispy-forms==1.14.0 # https://github.com/django-crispy-forms/django-cri crispy-bootstrap5==0.6 # https://github.com/django-crispy-forms/crispy-bootstrap5 # Django REST Framework -# djangorestframework==3.13.1 # https://github.com/encode/django-rest-framework +djangorestframework~=3.15 # https://github.com/encode/django-rest-framework django-cors-headers==3.13.0 # https://github.com/adamchainz/django-cors-headers # DRF-spectacular for api documentation drf-spectacular # https://github.com/tfranzel/drf-spectacular From a3e6c4e751e6f411372d1699dc4a43b60146f696 Mon Sep 17 00:00:00 2001 From: chriscummings Date: Thu, 14 Nov 2024 13:05:05 -0600 Subject: [PATCH 020/156] chore(deps): Remove more frozen downstream deps --- requirements/base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 50c1a84a..62756c66 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,7 +7,6 @@ celery==5.2.7 # pyup: < 6.0 # https://github.com/celery/celery django-celery-beat # https://github.com/celery/django-celery-beat flower==1.0.0 # https://github.com/mher/flower uvicorn[standard]==0.17.6 # https://github.com/encode/uvicorn -# asgiref~=3.5.0 # https://github.com/django/asgiref # Django # ------------------------------------------------------------------------------ From f22f2e813b1b0fe5d98dd77cd22aa668220c0100 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 15:32:12 -0600 Subject: [PATCH 021/156] refactor(auth): move auth_method to production which is the only file we use it in also appeases flake8 --- config/settings/production.py | 103 ++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/config/settings/production.py b/config/settings/production.py index 6d4484cc..e06d8fa2 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -1,3 +1,4 @@ +import logging import os from .base import * # noqa @@ -141,44 +142,64 @@ # Your stuff... # ------------------------------------------------------------------------------ -# Extend middleware to add OIDC middleware -MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 - -# Extend middleware to add OIDC auth backend -AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 - -# https://docs.djangoproject.com/en/dev/ref/settings/#login-url -LOGIN_URL = "oidc_authentication_init" - -# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url -LOGIN_REDIRECT_URL = "/" - -# https://docs.djangoproject.com/en/dev/ref/settings/#logout-url -LOGOUT_URL = "oidc_logout" - -# Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s -# https://github.com/mozilla/mozilla-django-oidc/issues/118 -# Using `/` because named urls don't work for this package -# https://github.com/mozilla/mozilla-django-oidc/issues/434 -LOGOUT_REDIRECT_URL = "/" - -OIDC_OP_JWKS_ENDPOINT = os.environ.get( - "OIDC_OP_JWKS_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/certs", -) -OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( - "OIDC_OP_AUTHORIZATION_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/auth", -) -OIDC_OP_TOKEN_ENDPOINT = os.environ.get( - "OIDC_OP_TOKEN_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/token", -) -OIDC_OP_USER_ENDPOINT = os.environ.get( - "OIDC_OP_USER_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", -) -OIDC_RP_SIGN_ALGO = "RS256" - -OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") -OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") +# Are you using local passwords or oidc? +AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() + +logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") +if AUTH_METHOD == "oidc": + # Extend middleware to add OIDC middleware + MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 + + # Extend middleware to add OIDC auth backend + AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url + LOGIN_URL = "oidc_authentication_init" + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url + LOGIN_REDIRECT_URL = "/" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url + LOGOUT_URL = "oidc_logout" + + # Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s + # https://github.com/mozilla/mozilla-django-oidc/issues/118 + # Using `/` because named urls don't work for this package + # https://github.com/mozilla/mozilla-django-oidc/issues/434 + LOGOUT_REDIRECT_URL = "/" + + OIDC_OP_JWKS_ENDPOINT = os.environ.get( + "OIDC_OP_JWKS_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/certs", + ) + OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( + "OIDC_OP_AUTHORIZATION_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/auth", + ) + OIDC_OP_TOKEN_ENDPOINT = os.environ.get( + "OIDC_OP_TOKEN_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/token", + ) + OIDC_OP_USER_ENDPOINT = os.environ.get( + "OIDC_OP_USER_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", + ) + OIDC_RP_SIGN_ALGO = "RS256" + + OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") + OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") + +elif AUTH_METHOD == "local": + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url + LOGIN_URL = "/login" + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url + LOGIN_REDIRECT_URL = "route_manager:home" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url + LOGOUT_URL = "/logout" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url + LOGOUT_REDIRECT_URL = "/" +else: + raise Exception(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") From d34334efa99284b786338ff21da0c19cab3d9dcd Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 15:35:16 -0600 Subject: [PATCH 022/156] style(isort): run isort on base.py --- config/settings/base.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index b9e1ca01..91e609e0 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -1,7 +1,6 @@ """ Base settings to build other settings files upon. """ -import os from pathlib import Path import environ @@ -97,12 +96,6 @@ ] # https://docs.djangoproject.com/en/dev/ref/settings/#auth-user-model AUTH_USER_MODEL = "users.User" -# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url -LOGIN_REDIRECT_URL = "route_manager:home" -# https://docs.djangoproject.com/en/dev/ref/settings/#login-url -LOGIN_URL = "admin:login" -# https://docs.djangoproject.com/en/dev/ref/settings/#logout-url -LOGOUT_URL = "admin:logout" # PASSWORDS # ------------------------------------------------------------------------------ @@ -293,9 +286,6 @@ SIMPLE_HISTORY_HISTORY_CHANGE_REASON_USE_TEXT_FIELD = True SIMPLE_HISTORY_ENABLED = True -# Are you using local passwords or oidc? -AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local") - # Users in these groups have full privileges, including Django is_superuser SCRAM_ADMIN_GROUPS = ["svc_scram_admin"] From d98363acced7118af2aee89a93d2e88dd2c2b131 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 15:35:55 -0600 Subject: [PATCH 023/156] refactor(settings): put auth settings for a make toggle-local situation in our local settings instead of inheriting them --- config/settings/local.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/settings/local.py b/config/settings/local.py index 14ec647b..9a6fd4de 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -67,3 +67,12 @@ # Behave Django testing framework INSTALLED_APPS += ["behave_django"] # noqa F405 + +# AUTHENTICATION +# ------------------------------------------------------------------------------ +# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url +LOGIN_REDIRECT_URL = "route_manager:home" +# https://docs.djangoproject.com/en/dev/ref/settings/#login-url +LOGIN_URL = "admin:login" +# https://docs.djangoproject.com/en/dev/ref/settings/#logout-url +LOGOUT_URL = "admin:logout" From ecd90d177088055e87d987adbd0c3554880012c7 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 16:11:16 -0600 Subject: [PATCH 024/156] style(precommit): ran hooks --- scram/local_auth/__init__.py | 0 scram/local_auth/urls.py | 9 +++++++++ scram/local_auth/views.py | 9 +++++++++ 3 files changed, 18 insertions(+) create mode 100644 scram/local_auth/__init__.py create mode 100644 scram/local_auth/urls.py create mode 100644 scram/local_auth/views.py diff --git a/scram/local_auth/__init__.py b/scram/local_auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scram/local_auth/urls.py b/scram/local_auth/urls.py new file mode 100644 index 00000000..0c8f7345 --- /dev/null +++ b/scram/local_auth/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from scram.local_auth.views import login, logout + +app_name = "users" +urlpatterns = [ + path("login/", view=login, name="update"), + path("logout/", view=logout, name="detail"), +] diff --git a/scram/local_auth/views.py b/scram/local_auth/views.py new file mode 100644 index 00000000..d4d0cb8c --- /dev/null +++ b/scram/local_auth/views.py @@ -0,0 +1,9 @@ +from django.shortcuts import render + + +def login(request): + return render(request, "account/login.html") + + +def logout(request): + return render(request, "account/logout.html") From 08625aa6770a27f5c0acb2d9c12605a31b21741f Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 16:12:16 -0600 Subject: [PATCH 025/156] feat(local_auth): set up the urls and point to them for the make toggle-prod with local auth use case --- config/settings/production.py | 6 +++--- config/urls.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/settings/production.py b/config/settings/production.py index e06d8fa2..258f4877 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -191,15 +191,15 @@ elif AUTH_METHOD == "local": # https://docs.djangoproject.com/en/dev/ref/settings/#login-url - LOGIN_URL = "/login" + LOGIN_URL = "local_auth:login" # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url LOGIN_REDIRECT_URL = "route_manager:home" # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url - LOGOUT_URL = "/logout" + LOGOUT_URL = "local_auth:logout" # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url - LOGOUT_REDIRECT_URL = "/" + LOGOUT_REDIRECT_URL = "route_manager:home" else: raise Exception(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") diff --git a/config/urls.py b/config/urls.py index a52cd002..ec9e5b7b 100644 --- a/config/urls.py +++ b/config/urls.py @@ -28,7 +28,8 @@ import mozilla_django_oidc # noqa: F401 urlpatterns += [path("oidc/", include("mozilla_django_oidc.urls"))] - +elif settings.AUTH_METHOD == "local": + urlpatterns += [path("auth/", include("scram.local_auth.urls", namespace="local_auth"))] # API URLS api_version_urls = ( [ From 2fc25a53509a33870095bbfb8364f491d30d569a Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 14 Nov 2024 16:14:24 -0600 Subject: [PATCH 026/156] refactor(formatting): fix formatting --- scram/templates/account/login.html | 25 +------------------------ scram/templates/account/logout.html | 3 +-- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/scram/templates/account/login.html b/scram/templates/account/login.html index 2cadea6a..ea2c5642 100644 --- a/scram/templates/account/login.html +++ b/scram/templates/account/login.html @@ -1,7 +1,6 @@ {% extends "account/base.html" %} {% load i18n %} -{% load account socialaccount %} {% load crispy_forms_tags %} {% block head_title %}{% trans "Sign In" %}{% endblock %} @@ -10,39 +9,17 @@

{% trans "Sign In" %}

-{% get_providers as socialaccount_providers %} -{% if socialaccount_providers %} -

{% blocktrans with site.name as site_name %}Please sign in with one -of your existing third party accounts. Or, sign up -for a {{ site_name }} account and sign in below:{% endblocktrans %}

- -
- -
    - {% include "socialaccount/snippets/provider_list.html" with process="login" %} -
- - - -
- -{% include "socialaccount/snippets/login_extra.html" %} - -{% else %}

{% blocktrans %}If you have not created an account yet, then please sign up first.{% endblocktrans %}

-{% endif %} - {% endblock %} - diff --git a/scram/templates/account/logout.html b/scram/templates/account/logout.html index 8e2e6754..6a6d9263 100644 --- a/scram/templates/account/logout.html +++ b/scram/templates/account/logout.html @@ -9,7 +9,7 @@

{% trans "Sign Out" %}

{% trans 'Are you sure you want to sign out?' %}

-
+ {% csrf_token %} {% if redirect_field_value %} @@ -19,4 +19,3 @@

{% trans "Sign Out" %}

{% endblock %} - From 39ba31cca13a903247b271f2b8e486f5dbbe78d8 Mon Sep 17 00:00:00 2001 From: chriscummings Date: Fri, 15 Nov 2024 09:41:20 -0600 Subject: [PATCH 027/156] ci(docs): only build on main --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1f7e14f0..d3fbb051 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,7 @@ name: Build sphinx docs on: push: branches: - - '**' + - 'main' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 3cb82d68df3eca6d18aa35578742c051fb10d9a2 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 11:10:04 -0600 Subject: [PATCH 028/156] style(hooks): ran precommit --- config/settings/production.py | 4 ++-- scram/local_auth/urls.py | 8 ++++---- scram/local_auth/views.py | 9 --------- scram/templates/account/login.html | 6 +++--- scram/templates/account/logout.html | 4 ++-- 5 files changed, 11 insertions(+), 20 deletions(-) delete mode 100644 scram/local_auth/views.py diff --git a/config/settings/production.py b/config/settings/production.py index 258f4877..ed5d707c 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -157,7 +157,7 @@ LOGIN_URL = "oidc_authentication_init" # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "/" + LOGIN_REDIRECT_URL = "route_manager:home" # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url LOGOUT_URL = "oidc_logout" @@ -166,7 +166,7 @@ # https://github.com/mozilla/mozilla-django-oidc/issues/118 # Using `/` because named urls don't work for this package # https://github.com/mozilla/mozilla-django-oidc/issues/434 - LOGOUT_REDIRECT_URL = "/" + LOGOUT_REDIRECT_URL = "route_manager:home" OIDC_OP_JWKS_ENDPOINT = os.environ.get( "OIDC_OP_JWKS_ENDPOINT", diff --git a/scram/local_auth/urls.py b/scram/local_auth/urls.py index 0c8f7345..a6fe2555 100644 --- a/scram/local_auth/urls.py +++ b/scram/local_auth/urls.py @@ -1,9 +1,9 @@ +from django.contrib.auth import views as auth_views from django.urls import path -from scram.local_auth.views import login, logout +app_name = "local_auth" -app_name = "users" urlpatterns = [ - path("login/", view=login, name="update"), - path("logout/", view=logout, name="detail"), + path("login/", auth_views.login, {"template_name": "account/login.html"}, name="login"), + path("logout/", auth_views.logout, {"template_name": "logged_out.html"}, name="logout"), ] diff --git a/scram/local_auth/views.py b/scram/local_auth/views.py deleted file mode 100644 index d4d0cb8c..00000000 --- a/scram/local_auth/views.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.shortcuts import render - - -def login(request): - return render(request, "account/login.html") - - -def logout(request): - return render(request, "account/logout.html") diff --git a/scram/templates/account/login.html b/scram/templates/account/login.html index ea2c5642..46db8b4c 100644 --- a/scram/templates/account/login.html +++ b/scram/templates/account/login.html @@ -1,11 +1,11 @@ -{% extends "account/base.html" %} +{% extends "base.html" %} {% load i18n %} {% load crispy_forms_tags %} {% block head_title %}{% trans "Sign In" %}{% endblock %} -{% block inner %} +{% block content %}

{% trans "Sign In" %}

@@ -13,7 +13,7 @@

{% trans "Sign In" %}

{% blocktrans %}If you have not created an account yet, then please sign up first.{% endblocktrans %}

- + {% csrf_token %} {{ form|crispy }} {% if redirect_field_value %} diff --git a/scram/templates/account/logout.html b/scram/templates/account/logout.html index 6a6d9263..2fcbdf09 100644 --- a/scram/templates/account/logout.html +++ b/scram/templates/account/logout.html @@ -1,10 +1,10 @@ -{% extends "account/base.html" %} +{% extends "base.html" %} {% load i18n %} {% block head_title %}{% trans "Sign Out" %}{% endblock %} -{% block inner %} +{% block content %}

{% trans "Sign Out" %}

{% trans 'Are you sure you want to sign out?' %}

From 4cc628ed2947dce568652f22fc7742986bed1cc8 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 12:38:25 -0600 Subject: [PATCH 029/156] feat(auth): set up the most basic working version of local auth use the django provided views and keep the template very basic for now. also remove unnecessary templates from the cookiecutter --- .env | 2 - scram/local_auth/urls.py | 10 ++- scram/templates/account/account_inactive.html | 12 --- scram/templates/account/base.html | 10 --- scram/templates/account/email.html | 80 ------------------- scram/templates/account/email_confirm.html | 32 -------- scram/templates/account/login.html | 25 ------ scram/templates/account/logout.html | 21 ----- scram/templates/account/password_change.html | 17 ---- scram/templates/account/password_reset.html | 25 ------ .../account/password_reset_done.html | 16 ---- .../account/password_reset_from_key.html | 24 ------ .../account/password_reset_from_key_done.html | 10 --- scram/templates/account/password_set.html | 17 ---- scram/templates/account/signup.html | 23 ------ scram/templates/account/signup_closed.html | 12 --- .../templates/account/verification_sent.html | 13 --- .../account/verified_email_required.html | 24 ------ scram/templates/local_auth/login.html | 12 +++ 19 files changed, 19 insertions(+), 366 deletions(-) delete mode 100644 .env delete mode 100644 scram/templates/account/account_inactive.html delete mode 100644 scram/templates/account/base.html delete mode 100644 scram/templates/account/email.html delete mode 100644 scram/templates/account/email_confirm.html delete mode 100644 scram/templates/account/login.html delete mode 100644 scram/templates/account/logout.html delete mode 100644 scram/templates/account/password_change.html delete mode 100644 scram/templates/account/password_reset.html delete mode 100644 scram/templates/account/password_reset_done.html delete mode 100644 scram/templates/account/password_reset_from_key.html delete mode 100644 scram/templates/account/password_reset_from_key_done.html delete mode 100644 scram/templates/account/password_set.html delete mode 100644 scram/templates/account/signup.html delete mode 100644 scram/templates/account/signup_closed.html delete mode 100644 scram/templates/account/verification_sent.html delete mode 100644 scram/templates/account/verified_email_required.html create mode 100644 scram/templates/local_auth/login.html diff --git a/.env b/.env deleted file mode 100644 index 3a5ce563..00000000 --- a/.env +++ /dev/null @@ -1,2 +0,0 @@ -CI_PROJECT_DIR=. -HOSTNAME=$(hostname) diff --git a/scram/local_auth/urls.py b/scram/local_auth/urls.py index a6fe2555..0d930946 100644 --- a/scram/local_auth/urls.py +++ b/scram/local_auth/urls.py @@ -1,9 +1,13 @@ -from django.contrib.auth import views as auth_views +from django.contrib.auth.views import LoginView, LogoutView from django.urls import path app_name = "local_auth" urlpatterns = [ - path("login/", auth_views.login, {"template_name": "account/login.html"}, name="login"), - path("logout/", auth_views.logout, {"template_name": "logged_out.html"}, name="logout"), + path( + "login/", + LoginView.as_view(template_name="local_auth/login.html", success_url="route_manager:home"), + name="login", + ), + path("logout/", LogoutView.as_view(), name="logout"), ] diff --git a/scram/templates/account/account_inactive.html b/scram/templates/account/account_inactive.html deleted file mode 100644 index 17c21577..00000000 --- a/scram/templates/account/account_inactive.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} - -{% block head_title %}{% trans "Account Inactive" %}{% endblock %} - -{% block inner %} -

{% trans "Account Inactive" %}

- -

{% trans "This account is inactive." %}

-{% endblock %} - diff --git a/scram/templates/account/base.html b/scram/templates/account/base.html deleted file mode 100644 index 97f4ddb8..00000000 --- a/scram/templates/account/base.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "../base.html" %} -{% block title %}{% block head_title %}{% endblock head_title %}{% endblock title %} - -{% block content %} -
-
- {% block inner %}{% endblock %} -
-
-{% endblock %} diff --git a/scram/templates/account/email.html b/scram/templates/account/email.html deleted file mode 100644 index 8eef4159..00000000 --- a/scram/templates/account/email.html +++ /dev/null @@ -1,80 +0,0 @@ - -{% extends "account/base.html" %} - -{% load i18n %} -{% load crispy_forms_tags %} - -{% block head_title %}{% trans "Account" %}{% endblock %} - -{% block inner %} -

{% trans "E-mail Addresses" %}

- -{% if user.emailaddress_set.all %} -

{% trans 'The following e-mail addresses are associated with your account:' %}

- - -{% csrf_token %} -
- - {% for emailaddress in user.emailaddress_set.all %} -
- -
- {% endfor %} - -
- - - -
- -
- - -{% else %} -

{% trans 'Warning:'%} {% trans "You currently do not have any e-mail address set up. You should really add an e-mail address so you can receive notifications, reset your password, etc." %}

- -{% endif %} - - -

{% trans "Add E-mail Address" %}

- -
- {% csrf_token %} - {{ form|crispy }} - -
- -{% endblock %} - - -{% block inline_javascript %} -{{ block.super }} - -{% endblock %} - diff --git a/scram/templates/account/email_confirm.html b/scram/templates/account/email_confirm.html deleted file mode 100644 index 46c78126..00000000 --- a/scram/templates/account/email_confirm.html +++ /dev/null @@ -1,32 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load account %} - -{% block head_title %}{% trans "Confirm E-mail Address" %}{% endblock %} - - -{% block inner %} -

{% trans "Confirm E-mail Address" %}

- -{% if confirmation %} - -{% user_display confirmation.email_address.user as user_display %} - -

{% blocktrans with confirmation.email_address.email as email %}Please confirm that {{ email }} is an e-mail address for user {{ user_display }}.{% endblocktrans %}

- -
-{% csrf_token %} - -
- -{% else %} - -{% url 'account_email' as email_url %} - -

{% blocktrans %}This e-mail confirmation link expired or is invalid. Please issue a new e-mail confirmation request.{% endblocktrans %}

- -{% endif %} - -{% endblock %} - diff --git a/scram/templates/account/login.html b/scram/templates/account/login.html deleted file mode 100644 index 46db8b4c..00000000 --- a/scram/templates/account/login.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "base.html" %} - -{% load i18n %} -{% load crispy_forms_tags %} - -{% block head_title %}{% trans "Sign In" %}{% endblock %} - -{% block content %} - -

{% trans "Sign In" %}

- - -

{% blocktrans %}If you have not created an account yet, then please -sign up first.{% endblocktrans %}

- - - -{% endblock %} diff --git a/scram/templates/account/logout.html b/scram/templates/account/logout.html deleted file mode 100644 index 2fcbdf09..00000000 --- a/scram/templates/account/logout.html +++ /dev/null @@ -1,21 +0,0 @@ -{% extends "base.html" %} - -{% load i18n %} - -{% block head_title %}{% trans "Sign Out" %}{% endblock %} - -{% block content %} -

{% trans "Sign Out" %}

- -

{% trans 'Are you sure you want to sign out?' %}

- -
- {% csrf_token %} - {% if redirect_field_value %} - - {% endif %} - -
- - -{% endblock %} diff --git a/scram/templates/account/password_change.html b/scram/templates/account/password_change.html deleted file mode 100644 index b72ca068..00000000 --- a/scram/templates/account/password_change.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load crispy_forms_tags %} - -{% block head_title %}{% trans "Change Password" %}{% endblock %} - -{% block inner %} -

{% trans "Change Password" %}

- -
- {% csrf_token %} - {{ form|crispy }} - -
-{% endblock %} - diff --git a/scram/templates/account/password_reset.html b/scram/templates/account/password_reset.html deleted file mode 100644 index c98f28c1..00000000 --- a/scram/templates/account/password_reset.html +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load account %} -{% load crispy_forms_tags %} - -{% block head_title %}{% trans "Password Reset" %}{% endblock %} - -{% block inner %} - -

{% trans "Password Reset" %}

- {% if user.is_authenticated %} - {% include "/snippets/already_logged_in.html" %} - {% endif %} - -

{% trans "Forgotten your password? Enter your e-mail address below, and we'll send you an e-mail allowing you to reset it." %}

- -
- {% csrf_token %} - {{ form|crispy }} - -
- -

{% blocktrans %}Please contact us if you have any trouble resetting your password.{% endblocktrans %}

-{% endblock %} diff --git a/scram/templates/account/password_reset_done.html b/scram/templates/account/password_reset_done.html deleted file mode 100644 index 835156ca..00000000 --- a/scram/templates/account/password_reset_done.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load account %} - -{% block head_title %}{% trans "Password Reset" %}{% endblock %} - -{% block inner %} -

{% trans "Password Reset" %}

- - {% if user.is_authenticated %} - {% include "/snippets/already_logged_in.html" %} - {% endif %} - -

{% blocktrans %}We have sent you an e-mail. Please contact us if you do not receive it within a few minutes.{% endblocktrans %}

-{% endblock %} diff --git a/scram/templates/account/password_reset_from_key.html b/scram/templates/account/password_reset_from_key.html deleted file mode 100644 index 2e2cd194..00000000 --- a/scram/templates/account/password_reset_from_key.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load crispy_forms_tags %} -{% block head_title %}{% trans "Change Password" %}{% endblock %} - -{% block inner %} -

{% if token_fail %}{% trans "Bad Token" %}{% else %}{% trans "Change Password" %}{% endif %}

- - {% if token_fail %} - {% url 'account_reset_password' as passwd_reset_url %} -

{% blocktrans %}The password reset link was invalid, possibly because it has already been used. Please request a new password reset.{% endblocktrans %}

- {% else %} - {% if form %} -
- {% csrf_token %} - {{ form|crispy }} - -
- {% else %} -

{% trans 'Your password is now changed.' %}

- {% endif %} - {% endif %} -{% endblock %} diff --git a/scram/templates/account/password_reset_from_key_done.html b/scram/templates/account/password_reset_from_key_done.html deleted file mode 100644 index 89be086f..00000000 --- a/scram/templates/account/password_reset_from_key_done.html +++ /dev/null @@ -1,10 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% block head_title %}{% trans "Change Password" %}{% endblock %} - -{% block inner %} -

{% trans "Change Password" %}

-

{% trans 'Your password is now changed.' %}

-{% endblock %} - diff --git a/scram/templates/account/password_set.html b/scram/templates/account/password_set.html deleted file mode 100644 index 22322235..00000000 --- a/scram/templates/account/password_set.html +++ /dev/null @@ -1,17 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load crispy_forms_tags %} - -{% block head_title %}{% trans "Set Password" %}{% endblock %} - -{% block inner %} -

{% trans "Set Password" %}

- -
- {% csrf_token %} - {{ form|crispy }} - -
-{% endblock %} - diff --git a/scram/templates/account/signup.html b/scram/templates/account/signup.html deleted file mode 100644 index 6a2954eb..00000000 --- a/scram/templates/account/signup.html +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} -{% load crispy_forms_tags %} - -{% block head_title %}{% trans "Signup" %}{% endblock %} - -{% block inner %} -

{% trans "Sign Up" %}

- -

{% blocktrans %}Already have an account? Then please sign in.{% endblocktrans %}

- - - -{% endblock %} - diff --git a/scram/templates/account/signup_closed.html b/scram/templates/account/signup_closed.html deleted file mode 100644 index 2322f176..00000000 --- a/scram/templates/account/signup_closed.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} - -{% block head_title %}{% trans "Sign Up Closed" %}{% endblock %} - -{% block inner %} -

{% trans "Sign Up Closed" %}

- -

{% trans "We are sorry, but the sign up is currently closed." %}

-{% endblock %} - diff --git a/scram/templates/account/verification_sent.html b/scram/templates/account/verification_sent.html deleted file mode 100644 index ad093fd4..00000000 --- a/scram/templates/account/verification_sent.html +++ /dev/null @@ -1,13 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} - -{% block head_title %}{% trans "Verify Your E-mail Address" %}{% endblock %} - -{% block inner %} -

{% trans "Verify Your E-mail Address" %}

- -

{% blocktrans %}We have sent an e-mail to you for verification. Follow the link provided to finalize the signup process. Please contact us if you do not receive it within a few minutes.{% endblocktrans %}

- -{% endblock %} - diff --git a/scram/templates/account/verified_email_required.html b/scram/templates/account/verified_email_required.html deleted file mode 100644 index 09d4fde7..00000000 --- a/scram/templates/account/verified_email_required.html +++ /dev/null @@ -1,24 +0,0 @@ -{% extends "account/base.html" %} - -{% load i18n %} - -{% block head_title %}{% trans "Verify Your E-mail Address" %}{% endblock %} - -{% block inner %} -

{% trans "Verify Your E-mail Address" %}

- -{% url 'account_email' as email_url %} - -

{% blocktrans %}This part of the site requires us to verify that -you are who you claim to be. For this purpose, we require that you -verify ownership of your e-mail address. {% endblocktrans %}

- -

{% blocktrans %}We have sent an e-mail to you for -verification. Please click on the link inside this e-mail. Please -contact us if you do not receive it within a few minutes.{% endblocktrans %}

- -

{% blocktrans %}Note: you can still change your e-mail address.{% endblocktrans %}

- - -{% endblock %} - diff --git a/scram/templates/local_auth/login.html b/scram/templates/local_auth/login.html new file mode 100644 index 00000000..e2355d27 --- /dev/null +++ b/scram/templates/local_auth/login.html @@ -0,0 +1,12 @@ +{% extends 'base.html' %} + +{% block title %}Login{% endblock %} + +{% block content %} +

Login

+
+ {% csrf_token %} + {{ form.as_p }} + +
+{% endblock %} From 95b921d43266deed804cd07a7f26911959a53c21 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 12:45:47 -0600 Subject: [PATCH 030/156] fix(gh-actions): pull in docs temp fix --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1f7e14f0..d3fbb051 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,7 @@ name: Build sphinx docs on: push: branches: - - '**' + - 'main' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 2074d72a300ba1095b5e182ded708ca49b74e7aa Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 12:53:28 -0600 Subject: [PATCH 031/156] fix(env-file): replace the .env file that i deleted thinking we didnt need but we sure do --- .env | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 00000000..3a5ce563 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +CI_PROJECT_DIR=. +HOSTNAME=$(hostname) From b6bfd9bc21e91eedfe359f927f55acb1790e36b0 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 13:04:43 -0600 Subject: [PATCH 032/156] style(hookes): ran precommit --- config/settings/base.py | 4 ++++ config/settings/production.py | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 91e609e0..26ed8ec8 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -1,6 +1,8 @@ """ Base settings to build other settings files upon. """ + +import os from pathlib import Path import environ @@ -274,6 +276,8 @@ CORS_URLS_REGEX = r"^/api/.*$" # Your stuff... # ------------------------------------------------------------------------------ +# Are you using local passwords or oidc? +AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() # Should we create an admin user for you AUTOCREATE_ADMIN = True diff --git a/config/settings/production.py b/config/settings/production.py index ed5d707c..0d1ef70f 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -2,7 +2,7 @@ import os from .base import * # noqa -from .base import AUTHENTICATION_BACKENDS, MIDDLEWARE, env +from .base import AUTH_METHOD, AUTHENTICATION_BACKENDS, MIDDLEWARE, env # GENERAL # ------------------------------------------------------------------------------ @@ -142,9 +142,6 @@ # Your stuff... # ------------------------------------------------------------------------------ -# Are you using local passwords or oidc? -AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() - logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware From 602767ecb574f4b5ea10c5fd18c2d3f3d9c9803d Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 13:19:40 -0600 Subject: [PATCH 033/156] refactor(hooks): ran precommit --- config/settings/test.py | 78 +++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 22 deletions(-) diff --git a/config/settings/test.py b/config/settings/test.py index 1eb5ed4a..9c89f01c 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -5,7 +5,7 @@ import os from .base import * # noqa -from .base import env +from .base import AUTH_METHOD, env # GENERAL # ------------------------------------------------------------------------------ @@ -41,26 +41,60 @@ # Your stuff... # ------------------------------------------------------------------------------ -# Extend middleware to add OIDC middleware -MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 +if AUTH_METHOD == "oidc": + # Extend middleware to add OIDC middleware + MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 -OIDC_OP_JWKS_ENDPOINT = os.environ.get( - "OIDC_OP_JWKS_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/certs", -) -OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( - "OIDC_OP_AUTHORIZATION_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/auth", -) -OIDC_OP_TOKEN_ENDPOINT = os.environ.get( - "OIDC_OP_TOKEN_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/token", -) -OIDC_OP_USER_ENDPOINT = os.environ.get( - "OIDC_OP_USER_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", -) -OIDC_RP_SIGN_ALGO = "RS256" + # Extend middleware to add OIDC auth backend + AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url + LOGIN_URL = "oidc_authentication_init" + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url + LOGIN_REDIRECT_URL = "route_manager:home" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url + LOGOUT_URL = "oidc_logout" + + # Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s + # https://github.com/mozilla/mozilla-django-oidc/issues/118 + # Using `/` because named urls don't work for this package + # https://github.com/mozilla/mozilla-django-oidc/issues/434 + LOGOUT_REDIRECT_URL = "route_manager:home" + + OIDC_OP_JWKS_ENDPOINT = os.environ.get( + "OIDC_OP_JWKS_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/certs", + ) + OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( + "OIDC_OP_AUTHORIZATION_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/auth", + ) + OIDC_OP_TOKEN_ENDPOINT = os.environ.get( + "OIDC_OP_TOKEN_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/token", + ) + OIDC_OP_USER_ENDPOINT = os.environ.get( + "OIDC_OP_USER_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", + ) + OIDC_RP_SIGN_ALGO = "RS256" + + OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") + OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") + +elif AUTH_METHOD == "local": + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url + LOGIN_URL = "local_auth:login" + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url + LOGIN_REDIRECT_URL = "route_manager:home" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url + LOGOUT_URL = "local_auth:logout" -OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID", "client_id") -OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET", "client_secret") + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url + LOGOUT_REDIRECT_URL = "route_manager:home" +else: + raise Exception(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") From 4fbfcb98dd4bd9136e0a7ada352dd2c0aa8acaa3 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 14:19:03 -0600 Subject: [PATCH 034/156] refactor(hooks): ran precommit --- config/settings/test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/settings/test.py b/config/settings/test.py index 9c89f01c..8d3a6cd5 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -41,6 +41,14 @@ # Your stuff... # ------------------------------------------------------------------------------ +OIDC_OP_JWKS_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/certs" +OIDC_OP_AUTHORIZATION_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/auth" +OIDC_OP_TOKEN_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/token" +OIDC_OP_USER_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/userinfo" +OIDC_RP_SIGN_ALGO = "RS256" +OIDC_RP_CLIENT_ID = "" +OIDC_RP_CLIENT_SECRET = "" + if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 From f0334b72d699504478388ce4b14f02830241a8b7 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 14:19:39 -0600 Subject: [PATCH 035/156] test(naming): update the name of the test case to better match reality --- scram/route_manager/tests/test_authorization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scram/route_manager/tests/test_authorization.py b/scram/route_manager/tests/test_authorization.py index 6f4269d1..cb4a52f4 100644 --- a/scram/route_manager/tests/test_authorization.py +++ b/scram/route_manager/tests/test_authorization.py @@ -125,7 +125,7 @@ def test_unauthorized_after_group_removal(self): self.assertEqual(response.status_code, 302) -class OidcTest(TestCase): +class ESnetAuthBackendTest(TestCase): def setUp(self): self.client = Client() self.claims = { From 29d29ab8a329dff8f3e86552691da74b39ced7b6 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 14:28:26 -0600 Subject: [PATCH 036/156] refactor(linting): should not be raising a raw exception --- config/settings/production.py | 2 +- config/settings/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/settings/production.py b/config/settings/production.py index 0d1ef70f..5e45125c 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -199,4 +199,4 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url LOGOUT_REDIRECT_URL = "route_manager:home" else: - raise Exception(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") + raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") diff --git a/config/settings/test.py b/config/settings/test.py index 8d3a6cd5..01f0cc8d 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -105,4 +105,4 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url LOGOUT_REDIRECT_URL = "route_manager:home" else: - raise Exception(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") + raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") From 6fa56482073d3430fe82b218e5d5f94dcccdec44 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 15 Nov 2024 20:05:09 -0600 Subject: [PATCH 037/156] feat(gobgp-neighbor): add makefile target for debugging gobgp neighbors --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 74556490..26b6f3a0 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,11 @@ down: compose.override.yml exec: compose.override.yml @docker compose exec $(CONTAINER) $(COMMAND) +## gobgp-neighbor: shows the gobgp neighbor information (append neighbor IP for specific information) +.Phony: gobgp-neighbor +gobgp-neighbor: compose.override.yml + @docker compose exec gobgp gobgp neighbor $(NEIGHBOR) + # This automatically builds the help target based on commands prepended with a double hashbang ## help: print this help output .Phony: help From ff716e87b56c84521aeed7464a7e821fd78d4e12 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 16 Nov 2024 10:32:53 -0600 Subject: [PATCH 038/156] Remove requirements that are pulled in by something else. --- requirements/base.txt | 22 +++++++++------------- requirements/local.txt | 7 ------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 62756c66..7577c200 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,26 +1,23 @@ -pytz==2022.1 # https://github.com/stub42/pytz -django-netfields # https://pypi.org/project/django-netfields/ -python-slugify==6.1.2 # https://github.com/un33k/python-slugify argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi -whitenoise==6.2.0 # https://github.com/evansd/whitenoise -celery==5.2.7 # pyup: < 6.0 # https://github.com/celery/celery django-celery-beat # https://github.com/celery/django-celery-beat +django-netfields # https://pypi.org/project/django-netfields/ flower==1.0.0 # https://github.com/mher/flower +python-slugify==6.1.2 # https://github.com/un33k/python-slugify uvicorn[standard]==0.17.6 # https://github.com/encode/uvicorn +whitenoise==6.2.0 # https://github.com/evansd/whitenoise # Django # ------------------------------------------------------------------------------ -channels==3.0.4 channels_redis==3.4.0 -django-redis==5.3.0 +crispy-bootstrap5==0.6 # https://github.com/django-crispy-forms/crispy-bootstrap5 django~=4.2 # https://www.djangoproject.com/ -django-grip==3.4.0 -django-eventstream==4.5.1 # https://github.com/fanout/django-eventstream + +django-allauth==0.51.0 # https://github.com/pennersr/django-allauth django-environ==0.9.0 # https://github.com/joke2k/django-environ +django-eventstream==4.5.1 # https://github.com/fanout/django-eventstream django-model-utils==4.2.0 # https://github.com/jazzband/django-model-utils -django-allauth==0.51.0 # https://github.com/pennersr/django-allauth -django-crispy-forms==1.14.0 # https://github.com/django-crispy-forms/django-crispy-forms -crispy-bootstrap5==0.6 # https://github.com/django-crispy-forms/crispy-bootstrap5 +django-redis==5.3.0 +django-simple-history~=3.1.1 # Django REST Framework djangorestframework~=3.15 # https://github.com/encode/django-rest-framework @@ -33,4 +30,3 @@ drf-spectacular # https://github.com/tfranzel/drf-spectacular mozilla_django_oidc==2.0.0 # https://github.com/mozilla/mozilla-django-oidc websockets~=10.3 -django-simple-history~=3.1.1 diff --git a/requirements/local.txt b/requirements/local.txt index edc15a30..a0fbe77b 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -7,12 +7,9 @@ watchgod==0.8.2 # https://github.com/samuelcolvin/watchgod # Testing # ------------------------------------------------------------------------------ -mypy==0.950 # https://github.com/python/mypy django-stubs==1.11.0 # https://github.com/typeddjango/django-stubs -pytest==7.1.2 # https://github.com/pytest-dev/pytest pytest-sugar==0.9.4 # https://github.com/Frozenball/pytest-sugar behave-django==1.4.0 # https://github.com/behave/behave-django -django-debug-toolbar~=3.2 # https://github.com/jazzband/django-debug-toolbar # Documentation # ------------------------------------------------------------------------------ @@ -21,9 +18,7 @@ sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild # Code quality # ------------------------------------------------------------------------------ -flake8==4.0.1 # https://github.com/PyCQA/flake8 flake8-isort==4.1.1 # https://github.com/gforcada/flake8-isort -coverage==6.4.1 # https://github.com/nedbat/coveragepy black==22.3.0 # https://github.com/psf/black pylint-django==2.5.3 # https://github.com/PyCQA/pylint-django pylint-celery==0.3 # https://github.com/PyCQA/pylint-celery @@ -37,8 +32,6 @@ django-debug-toolbar==3.4.0 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.5 # https://github.com/django-extensions/django-extensions django-coverage-plugin==2.0.3 # https://github.com/nedbat/django_coverage_plugin pytest-django==4.5.2 # https://github.com/pytest-dev/pytest-django -pytest~=7.1.2 -behave~=1.2.6 # https://behave.readthedocs.io/en/stable/ # Debugging # ------------------------------------------------------------------------------ From ebd3c7a41b85f2ee6c9ddcd5e97760e8b6706f01 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 09:07:36 -0600 Subject: [PATCH 039/156] First step to add coveralls --- .github/workflows/pytest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0073cd2e..1ea68e78 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -59,7 +59,10 @@ jobs: POSTGRES_DB: test_scram run: make coverage.xml - - name: Upload Coverage Report + - name: Upload Coverage to Coveralls + uses: coverallsapp/github-action@v2 + + - name: Upload Coverage to GitHub uses: actions/upload-artifact@v4 with: name: coverage-report From 894cef5754dd96864d008104daf005c3156c9480 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 09:09:12 -0600 Subject: [PATCH 040/156] YAML is so fun; fixing indent --- .github/workflows/pytest.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1ea68e78..f84957df 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,3 +1,4 @@ +--- name: Run pytest on: @@ -59,10 +60,10 @@ jobs: POSTGRES_DB: test_scram run: make coverage.xml - - name: Upload Coverage to Coveralls - uses: coverallsapp/github-action@v2 + - name: Upload Coverage to Coveralls + uses: coverallsapp/github-action@v2 - - name: Upload Coverage to GitHub + - name: Upload Coverage to GitHub uses: actions/upload-artifact@v4 with: name: coverage-report From 21cbe260af878597250018f29472f3ccab5889aa Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 11:38:42 -0600 Subject: [PATCH 041/156] Ignore some things from coverage stats --- config/asgi.py | 12 ++++++------ setup.cfg | 10 ++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/config/asgi.py b/config/asgi.py index 1b3cd6e9..98a16ce2 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -19,10 +19,10 @@ from django.core.asgi import get_asgi_application # Here we setup a debugger if this is desired. This obviously should not be run in production. -debug_mode = os.environ.get("DEBUG") -if debug_mode: - logging.info(f"Django is set to use a debugger. Provided debug mode: {debug_mode}") - if debug_mode == "pycharm-pydevd": +debug = os.environ.get("DEBUG") +if debug: + logging.info(f"Django is set to use a debugger. Provided debug mode: {debug}") + if debug == "pycharm-pydevd": logging.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") import pydevd_pycharm @@ -30,7 +30,7 @@ pydevd_pycharm.settrace("host.docker.internal", port=56783, stdoutToServer=True, stderrToServer=True) logging.info("Debugger started.") - elif debug_mode == "debugpy": + elif debug == "debugpy": logging.info("Entering debug mode for debugpy (VSCode)") import debugpy @@ -39,7 +39,7 @@ logging.info("Debugger listening on port 56780.") else: - logging.warning(f"Invalid debug mode given: {debug_mode}. Debugger not started") + logging.warning(f"Invalid debug mode given: {debug}. Debugger not started") # This allows easy placement of apps within the interior # scram directory. diff --git a/setup.cfg b/setup.cfg index 2c082581..6243121a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,3 +27,13 @@ ignore_errors = True [behave] paths = scram/route_manager/tests/acceptance stderr_capture = no + +[coverage:report] +exclude_also = + def __repr__ + if debug: + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if __name__ == .__main__.: From 41aca9a1cb6d957674d3e43a958c5573b67e3dcc Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 12:25:31 -0600 Subject: [PATCH 042/156] Coverage fixes: move settings to a single file, update version, fix configuration for django templates. --- .coveragerc | 3 --- config/settings/local.py | 4 ++++ pyproject.toml | 3 ++- requirements/local.txt | 2 +- setup.cfg | 1 - 5 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 457c5db9..00000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -branch = True -data_file = coverage.coverage diff --git a/config/settings/local.py b/config/settings/local.py index 14ec647b..4bc4ba57 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -33,6 +33,10 @@ # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS # noqa F405 +# django-coverage-plugin +# ------------------------------------------------------------------------------ +# https://github.com/nedbat/django_coverage_plugin?tab=readme-ov-file#django-template-coveragepy-plugin +TEMPLATES[0]["OPTIONS"]['debug'] = True # django-debug-toolbar # ------------------------------------------------------------------------------ diff --git a/pyproject.toml b/pyproject.toml index f5e74ec1..5981b6f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,8 @@ python_files = [ include = ["scram/*", "config/*", "translator/*"] omit = ["**/migrations/*", "scram/contrib/*", "*/tests/*"] plugins = ["django_coverage_plugin"] - +branch = true +data_file = coverage.coverage # ==== black ==== diff --git a/requirements/local.txt b/requirements/local.txt index a0fbe77b..6b6f0988 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -30,7 +30,7 @@ factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy django-debug-toolbar==3.4.0 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.5 # https://github.com/django-extensions/django-extensions -django-coverage-plugin==2.0.3 # https://github.com/nedbat/django_coverage_plugin +django-coverage-plugin==3.5.0 # https://github.com/nedbat/django_coverage_plugin pytest-django==4.5.2 # https://github.com/pytest-dev/pytest-django # Debugging diff --git a/setup.cfg b/setup.cfg index 6243121a..09cd74c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,6 @@ stderr_capture = no [coverage:report] exclude_also = - def __repr__ if debug: if self.debug: if settings.DEBUG From 310d7896a79219a8e947e8698a5b92313989b685 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 12:46:25 -0600 Subject: [PATCH 043/156] Remove the config star imports; there's really not that many of them. --- config/settings/local.py | 12 ++++++------ config/settings/production.py | 20 ++++++++++---------- config/settings/test.py | 6 +++--- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/config/settings/local.py b/config/settings/local.py index 4bc4ba57..a5ababf9 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -1,4 +1,4 @@ -from .base import * # noqa +from .base import INSTALLED_APPS, MIDDLEWARE, TEMPLATES from .base import env # GENERAL @@ -31,7 +31,7 @@ # WhiteNoise # ------------------------------------------------------------------------------ # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development -INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS # noqa F405 +INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS # django-coverage-plugin # ------------------------------------------------------------------------------ @@ -41,9 +41,9 @@ # django-debug-toolbar # ------------------------------------------------------------------------------ # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#prerequisites -INSTALLED_APPS += ["debug_toolbar"] # noqa F405 +INSTALLED_APPS += ["debug_toolbar"] # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware -MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] # noqa F405 +MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] # https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config DEBUG_TOOLBAR_CONFIG = { "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"], @@ -60,7 +60,7 @@ # django-extensions # ------------------------------------------------------------------------------ # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration -INSTALLED_APPS += ["django_extensions"] # noqa F405 +INSTALLED_APPS += ["django_extensions"] # Your stuff... # ------------------------------------------------------------------------------ @@ -70,4 +70,4 @@ } # Behave Django testing framework -INSTALLED_APPS += ["behave_django"] # noqa F405 +INSTALLED_APPS += ["behave_django"] diff --git a/config/settings/production.py b/config/settings/production.py index 6d4484cc..248f4a23 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -1,7 +1,7 @@ import os -from .base import * # noqa -from .base import AUTHENTICATION_BACKENDS, MIDDLEWARE, env +from .base import AUTHENTICATION_BACKENDS, DATABASES, INSTALLED_APPS, MIDDLEWARE, TEMPLATES +from .base import env # GENERAL # ------------------------------------------------------------------------------ @@ -12,11 +12,11 @@ # DATABASES # ------------------------------------------------------------------------------ -DATABASES["default"] = env.db("DATABASE_URL") # noqa F405 -DATABASES["default"]["ATOMIC_REQUESTS"] = True # noqa F405 -DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) # noqa F405 +DATABASES["default"] = env.db("DATABASE_URL") +DATABASES["default"]["ATOMIC_REQUESTS"] = True +DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) if env("POSTGRES_SSL"): - DATABASES["default"]["OPTIONS"] = {"sslmode": "require"} # noqa F405 + DATABASES["default"]["OPTIONS"] = {"sslmode": "require"} # CACHES # ------------------------------------------------------------------------------ @@ -63,7 +63,7 @@ # TEMPLATES # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#templates -TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] # noqa F405 +TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] ( "django.template.loaders.cached.Loader", [ @@ -90,7 +90,7 @@ # Anymail # ------------------------------------------------------------------------------ # https://anymail.readthedocs.io/en/stable/installation/#installing-anymail -INSTALLED_APPS += ["anymail"] # noqa F405 +INSTALLED_APPS += ["anymail"] # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend # https://anymail.readthedocs.io/en/stable/installation/#anymail-settings-reference # https://anymail.readthedocs.io/en/stable/esps @@ -142,10 +142,10 @@ # Your stuff... # ------------------------------------------------------------------------------ # Extend middleware to add OIDC middleware -MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 +MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # Extend middleware to add OIDC auth backend -AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 +AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "oidc_authentication_init" diff --git a/config/settings/test.py b/config/settings/test.py index 1eb5ed4a..6c5b06f5 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -4,7 +4,7 @@ import os -from .base import * # noqa +from .base import MIDDLEWARE, TEMPLATES from .base import env # GENERAL @@ -24,7 +24,7 @@ # TEMPLATES # ------------------------------------------------------------------------------ -TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] # noqa F405 +TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] ( "django.template.loaders.cached.Loader", [ @@ -42,7 +42,7 @@ # Your stuff... # ------------------------------------------------------------------------------ # Extend middleware to add OIDC middleware -MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 +MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] OIDC_OP_JWKS_ENDPOINT = os.environ.get( "OIDC_OP_JWKS_ENDPOINT", From dd12466a85b18761702aacfdaab981519e9e06d8 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 12:54:42 -0600 Subject: [PATCH 044/156] Fix django-coverage-plugin version --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index 6b6f0988..11b526f1 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -30,7 +30,7 @@ factory-boy==3.2.1 # https://github.com/FactoryBoy/factory_boy django-debug-toolbar==3.4.0 # https://github.com/jazzband/django-debug-toolbar django-extensions==3.1.5 # https://github.com/django-extensions/django-extensions -django-coverage-plugin==3.5.0 # https://github.com/nedbat/django_coverage_plugin +django-coverage-plugin==3.1.0 # https://github.com/nedbat/django_coverage_plugin pytest-django==4.5.2 # https://github.com/pytest-dev/pytest-django # Debugging From 72d40b147760e0ae4886b40ec9a71dc193a49d08 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 13:16:51 -0600 Subject: [PATCH 045/156] Bring syntax inline with the rest --- config/settings/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings/local.py b/config/settings/local.py index a5ababf9..1d781593 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -31,7 +31,7 @@ # WhiteNoise # ------------------------------------------------------------------------------ # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development -INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS +INSTALLED_APPS += ["whitenoise.runserver_nostatic"] # django-coverage-plugin # ------------------------------------------------------------------------------ From 5a68c46718043cdb69fdb4a2703d4f12bad8455d Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 13:27:19 -0600 Subject: [PATCH 046/156] Reverting 3 commits. They introduced a test failure, and I couldn't figure out how to resolve it. Leaving this as future work, and fixing the original issue (that those commits were trying to solve) a different way. --- config/settings/local.py | 14 +++++++------- config/settings/production.py | 20 ++++++++++---------- config/settings/test.py | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/config/settings/local.py b/config/settings/local.py index 1d781593..aaf58204 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -1,4 +1,4 @@ -from .base import INSTALLED_APPS, MIDDLEWARE, TEMPLATES +from .base import * # noqa from .base import env # GENERAL @@ -31,19 +31,19 @@ # WhiteNoise # ------------------------------------------------------------------------------ # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development -INSTALLED_APPS += ["whitenoise.runserver_nostatic"] +INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS # noqa F405 # django-coverage-plugin # ------------------------------------------------------------------------------ # https://github.com/nedbat/django_coverage_plugin?tab=readme-ov-file#django-template-coveragepy-plugin -TEMPLATES[0]["OPTIONS"]['debug'] = True +TEMPLATES[0]["OPTIONS"]['debug'] = True # noqa F405 # django-debug-toolbar # ------------------------------------------------------------------------------ # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#prerequisites -INSTALLED_APPS += ["debug_toolbar"] +INSTALLED_APPS += ["debug_toolbar"] # noqa F405 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware -MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] +MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] # noqa F405 # https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config DEBUG_TOOLBAR_CONFIG = { "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"], @@ -60,7 +60,7 @@ # django-extensions # ------------------------------------------------------------------------------ # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration -INSTALLED_APPS += ["django_extensions"] +INSTALLED_APPS += ["django_extensions"] # noqa F405 # Your stuff... # ------------------------------------------------------------------------------ @@ -70,4 +70,4 @@ } # Behave Django testing framework -INSTALLED_APPS += ["behave_django"] +INSTALLED_APPS += ["behave_django"] # noqa F405 diff --git a/config/settings/production.py b/config/settings/production.py index 248f4a23..6d4484cc 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -1,7 +1,7 @@ import os -from .base import AUTHENTICATION_BACKENDS, DATABASES, INSTALLED_APPS, MIDDLEWARE, TEMPLATES -from .base import env +from .base import * # noqa +from .base import AUTHENTICATION_BACKENDS, MIDDLEWARE, env # GENERAL # ------------------------------------------------------------------------------ @@ -12,11 +12,11 @@ # DATABASES # ------------------------------------------------------------------------------ -DATABASES["default"] = env.db("DATABASE_URL") -DATABASES["default"]["ATOMIC_REQUESTS"] = True -DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) +DATABASES["default"] = env.db("DATABASE_URL") # noqa F405 +DATABASES["default"]["ATOMIC_REQUESTS"] = True # noqa F405 +DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) # noqa F405 if env("POSTGRES_SSL"): - DATABASES["default"]["OPTIONS"] = {"sslmode": "require"} + DATABASES["default"]["OPTIONS"] = {"sslmode": "require"} # noqa F405 # CACHES # ------------------------------------------------------------------------------ @@ -63,7 +63,7 @@ # TEMPLATES # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#templates -TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] +TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] # noqa F405 ( "django.template.loaders.cached.Loader", [ @@ -90,7 +90,7 @@ # Anymail # ------------------------------------------------------------------------------ # https://anymail.readthedocs.io/en/stable/installation/#installing-anymail -INSTALLED_APPS += ["anymail"] +INSTALLED_APPS += ["anymail"] # noqa F405 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend # https://anymail.readthedocs.io/en/stable/installation/#anymail-settings-reference # https://anymail.readthedocs.io/en/stable/esps @@ -142,10 +142,10 @@ # Your stuff... # ------------------------------------------------------------------------------ # Extend middleware to add OIDC middleware -MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] +MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 # Extend middleware to add OIDC auth backend -AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] +AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "oidc_authentication_init" diff --git a/config/settings/test.py b/config/settings/test.py index 6c5b06f5..1eb5ed4a 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -4,7 +4,7 @@ import os -from .base import MIDDLEWARE, TEMPLATES +from .base import * # noqa from .base import env # GENERAL @@ -24,7 +24,7 @@ # TEMPLATES # ------------------------------------------------------------------------------ -TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] +TEMPLATES[-1]["OPTIONS"]["loaders"] = [ # type: ignore[index] # noqa F405 ( "django.template.loaders.cached.Loader", [ @@ -42,7 +42,7 @@ # Your stuff... # ------------------------------------------------------------------------------ # Extend middleware to add OIDC middleware -MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] +MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 OIDC_OP_JWKS_ENDPOINT = os.environ.get( "OIDC_OP_JWKS_ENDPOINT", From f8b9bff77591d1ade63646340d290688e5d1df11 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 13:41:16 -0600 Subject: [PATCH 047/156] Add Coveralls badge --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9be49463..0b5ed230 100644 --- a/README.rst +++ b/README.rst @@ -3,6 +3,9 @@ SCRAM Security Catch and Release Automation Manager +.. image:: https://coveralls.io/repos/github/esnet-security/SCRAM/badge.svg + :target: https://coveralls.io/github/esnet-security/SCRAM + :alt: Coveralls Code Coverage Stats .. image:: https://img.shields.io/badge/built%20with-Cookiecutter%20Django-ff69b4.svg?logo=cookiecutter :target: https://github.com/pydanny/cookiecutter-django/ :alt: Built with Cookiecutter Django @@ -10,7 +13,6 @@ Security Catch and Release Automation Manager :target: https://github.com/ambv/black :alt: Black code style - :License: BSD ==== From 95f34d1ffdab8421ac5c617a3b687e6fe49f174b Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 13:43:36 -0600 Subject: [PATCH 048/156] Consolidate coverage config --- pyproject.toml | 8 ++++++++ setup.cfg | 9 --------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5981b6f3..768016a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,14 @@ plugins = ["django_coverage_plugin"] branch = true data_file = coverage.coverage +[tool.coverage.report] +exclude_also = + if debug: + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if __name__ == .__main__.: # ==== black ==== [tool.black] diff --git a/setup.cfg b/setup.cfg index 09cd74c2..2c082581 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,12 +27,3 @@ ignore_errors = True [behave] paths = scram/route_manager/tests/acceptance stderr_capture = no - -[coverage:report] -exclude_also = - if debug: - if self.debug: - if settings.DEBUG - raise AssertionError - raise NotImplementedError - if __name__ == .__main__.: From 5fecad96b378a46d0b7ff9a2b87598c1b2ef9214 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 13:55:13 -0600 Subject: [PATCH 049/156] Fix toml syntax --- pyproject.toml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 768016a0..910571d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,16 +13,17 @@ include = ["scram/*", "config/*", "translator/*"] omit = ["**/migrations/*", "scram/contrib/*", "*/tests/*"] plugins = ["django_coverage_plugin"] branch = true -data_file = coverage.coverage +data_file = "coverage.coverage" [tool.coverage.report] -exclude_also = - if debug: - if self.debug: - if settings.DEBUG - raise AssertionError - raise NotImplementedError - if __name__ == .__main__.: +exclude_also = [ + "if debug:", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + ] # ==== black ==== [tool.black] From 58ebc6ffccfccc98450ae7b6a077b0c7479779a6 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 15:32:01 -0600 Subject: [PATCH 050/156] Add docstrings to scram/ --- scram/__init__.py | 4 ++- scram/conftest.py | 4 +++ scram/contrib/__init__.py | 2 +- scram/contrib/sites/__init__.py | 2 +- scram/route_manager/__init__.py | 1 + scram/route_manager/admin.py | 4 +++ scram/route_manager/api/__init__.py | 1 + scram/route_manager/api/exceptions.py | 8 +++++ scram/route_manager/api/serializers.py | 24 ++++++++++++++ scram/route_manager/api/views.py | 22 +++++++++++-- scram/route_manager/apps.py | 4 +++ .../route_manager/authentication_backends.py | 9 ++++-- scram/route_manager/context_processors.py | 3 ++ scram/route_manager/models.py | 28 ++++++++++++---- scram/route_manager/tests/__init__.py | 1 + .../tests/acceptance/environment.py | 3 ++ .../tests/acceptance/steps/common.py | 21 ++++++++++++ .../tests/acceptance/steps/ip.py | 7 +++- .../tests/acceptance/steps/translator.py | 5 +++ scram/route_manager/tests/functional_tests.py | 3 ++ scram/route_manager/tests/test_api.py | 15 +++++++++ .../route_manager/tests/test_authorization.py | 32 +++++++++++-------- scram/route_manager/tests/test_history.py | 10 ++++++ scram/route_manager/tests/test_views.py | 5 +++ scram/route_manager/tests/test_websockets.py | 14 ++++++-- scram/route_manager/urls.py | 2 ++ scram/route_manager/views.py | 12 +++++++ scram/users/__init__.py | 1 + scram/users/admin.py | 3 ++ scram/users/api/serializers.py | 6 ++++ scram/users/api/views.py | 6 ++++ scram/users/apps.py | 5 +++ scram/users/forms.py | 10 ++++++ scram/users/models.py | 2 ++ scram/users/tests/__init__.py | 1 + scram/users/tests/factories.py | 6 ++++ scram/users/tests/test_admin.py | 8 +++++ scram/users/tests/test_drf_urls.py | 5 +++ scram/users/tests/test_drf_views.py | 6 ++++ scram/users/tests/test_forms.py | 13 +++----- scram/users/tests/test_models.py | 3 ++ scram/users/tests/test_urls.py | 5 +++ scram/users/tests/test_views.py | 14 ++++++++ scram/users/urls.py | 2 ++ scram/users/views.py | 8 +++++ scram/utils/__init__.py | 1 + scram/utils/context_processors.py | 4 ++- 47 files changed, 316 insertions(+), 39 deletions(-) diff --git a/scram/__init__.py b/scram/__init__.py index eed836da..d5feb05a 100644 --- a/scram/__init__.py +++ b/scram/__init__.py @@ -1,2 +1,4 @@ -__version__ = "0.1.0" +"""The Django project for Security Catch and Release Automation Manager (SCRAM).""" + +__version__ = "1.1.1" __version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace("-", ".", 1).split(".")]) diff --git a/scram/conftest.py b/scram/conftest.py index 52d755b4..29a5bc61 100644 --- a/scram/conftest.py +++ b/scram/conftest.py @@ -1,3 +1,5 @@ +"""Some simple tests for the User app.""" + import pytest from scram.users.models import User @@ -6,9 +8,11 @@ @pytest.fixture(autouse=True) def media_storage(settings, tmpdir): + """Configure the test to use the temp directory.""" settings.MEDIA_ROOT = tmpdir.strpath @pytest.fixture def user() -> User: + """Return the UserFactory.""" return UserFactory() diff --git a/scram/contrib/__init__.py b/scram/contrib/__init__.py index 1c7ecc89..88d92bd3 100644 --- a/scram/contrib/__init__.py +++ b/scram/contrib/__init__.py @@ -1,5 +1,5 @@ """ -To understand why this file is here, please read: +To understand why this file is here, please read the CookieCutter documentation. http://cookiecutter-django.readthedocs.io/en/latest/faq.html#why-is-there-a-django-contrib-sites-directory-in-cookiecutter-django """ diff --git a/scram/contrib/sites/__init__.py b/scram/contrib/sites/__init__.py index 1c7ecc89..88d92bd3 100644 --- a/scram/contrib/sites/__init__.py +++ b/scram/contrib/sites/__init__.py @@ -1,5 +1,5 @@ """ -To understand why this file is here, please read: +To understand why this file is here, please read the CookieCutter documentation. http://cookiecutter-django.readthedocs.io/en/latest/faq.html#why-is-there-a-django-contrib-sites-directory-in-cookiecutter-django """ diff --git a/scram/route_manager/__init__.py b/scram/route_manager/__init__.py index e69de29b..db37df27 100644 --- a/scram/route_manager/__init__.py +++ b/scram/route_manager/__init__.py @@ -0,0 +1 @@ +"""Route Manager is the core app, and it handles adding and removing routes that will be blocked.""" diff --git a/scram/route_manager/admin.py b/scram/route_manager/admin.py index 96a6e324..172dd6c7 100644 --- a/scram/route_manager/admin.py +++ b/scram/route_manager/admin.py @@ -1,3 +1,5 @@ +"""Register models in the Admin site.""" + from django.contrib import admin from simple_history.admin import SimpleHistoryAdmin @@ -6,6 +8,8 @@ @admin.register(ActionType) class ActionTypeAdmin(SimpleHistoryAdmin): + """Configure the ActionType and how it shows up in the Admin site.""" + list_filter = ("available",) list_display = ("name", "available") diff --git a/scram/route_manager/api/__init__.py b/scram/route_manager/api/__init__.py index e69de29b..26fc9ceb 100644 --- a/scram/route_manager/api/__init__.py +++ b/scram/route_manager/api/__init__.py @@ -0,0 +1 @@ +"""The API, which leverages Django Request Framework.""" diff --git a/scram/route_manager/api/exceptions.py b/scram/route_manager/api/exceptions.py index 5483055b..e75c87b9 100644 --- a/scram/route_manager/api/exceptions.py +++ b/scram/route_manager/api/exceptions.py @@ -1,8 +1,12 @@ +"""Custom exceptions for the API.""" + from django.conf import settings from rest_framework.exceptions import APIException class PrefixTooLarge(APIException): + """The CIDR prefix that was specified is larger than 32 bits for IPv4 of 128 bits for IPv6.""" + v4_min_prefix = getattr(settings, "V4_MINPREFIX", 0) v6_min_prefix = getattr(settings, "V6_MINPREFIX", 0) status_code = 400 @@ -11,12 +15,16 @@ class PrefixTooLarge(APIException): class IgnoredRoute(APIException): + """An operation attempted to add a route that overlaps with a route on the ignore list.""" + status_code = 400 default_detail = "This CIDR is on the ignore list. You are not allowed to add it here." default_code = "ignored_route" class ActiontypeNotAllowed(APIException): + """An operation attempted to perform an action on behalf of a client that is unauthorized to perform that type.""" + status_code = 403 default_detail = "This client is not allowed to use this actiontype" default_code = "actiontype_not_allowed" diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index dec9df15..6baf7d95 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -1,3 +1,5 @@ +"""Serializers provide mappings between the API and the underlying model.""" + import logging from netfields import rest_framework @@ -11,15 +13,23 @@ class ActionTypeSerializer(serializers.ModelSerializer): + """This serializer defines no new fields.""" + class Meta: + """Maps to the ActionType model, and specifies the fields exposed by the API.""" + model = ActionType fields = ["pk", "name", "available"] class RouteSerializer(serializers.ModelSerializer): + """Exposes route as a CIDR field.""" + route = rest_framework.CidrAddressField() class Meta: + """Maps to the Route model, and specifies the fields exposed by the API.""" + model = Route fields = [ "route", @@ -27,12 +37,18 @@ class Meta: class ClientSerializer(serializers.ModelSerializer): + """This serializer defines no new fields.""" + class Meta: + """Maps to the Client model, and specifies the fields exposed by the API.""" + model = Client fields = ["hostname", "uuid"] class EntrySerializer(serializers.HyperlinkedModelSerializer): + """Due to the use of ForeignKeys, this follows some relationships to make sense via the API.""" + url = serializers.HyperlinkedIdentityField( view_name="api:v1:entry-detail", lookup_url_kwarg="pk", lookup_field="route" ) @@ -46,13 +62,17 @@ class EntrySerializer(serializers.HyperlinkedModelSerializer): comment = serializers.CharField() class Meta: + """Maps to the Entry model, and specifies the fields exposed by the API.""" + model = Entry fields = ["route", "actiontype", "url", "comment", "who"] def get_comment(self, obj): + """Provide a nicer name for change reason.""" return obj.get_change_reason() def create(self, validated_data): + """Implement custom logic and validates creating a new route.""" valid_route = validated_data.pop("route") actiontype = validated_data.pop("actiontype") comment = validated_data.pop("comment") @@ -68,6 +88,10 @@ def create(self, validated_data): class IgnoreEntrySerializer(serializers.ModelSerializer): + """This serializer defines no new fields.""" + class Meta: + """Maps to the IgnoreEntry model, and specifies the fields exposed by the API.""" + model = IgnoreEntry fields = ["route", "comment"] diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 2972e6dd..ca3a5d09 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -1,3 +1,5 @@ +"""Views provide mappings between the underlying model and how they're listed in the API.""" + import ipaddress import logging @@ -20,6 +22,8 @@ class ActionTypeViewSet(viewsets.ReadOnlyModelViewSet): + """Lookup ActionTypes by name when authenticated, and bind to the serializer.""" + queryset = ActionType.objects.all() permission_classes = (IsAuthenticated,) serializer_class = ActionTypeSerializer @@ -27,6 +31,8 @@ class ActionTypeViewSet(viewsets.ReadOnlyModelViewSet): class IgnoreEntryViewSet(viewsets.ModelViewSet): + """Lookup IgnoreEntries by route when authenticated, and bind to the serializer.""" + queryset = IgnoreEntry.objects.all() permission_classes = (IsAuthenticated,) serializer_class = IgnoreEntrySerializer @@ -34,6 +40,8 @@ class IgnoreEntryViewSet(viewsets.ModelViewSet): class ClientViewSet(viewsets.ModelViewSet): + """Lookup Client by hostname on POSTs regardless of authentication, and bind to the serializer.""" + queryset = Client.objects.all() # We want to allow a client to be registered from anywhere permission_classes = (AllowAny,) @@ -43,20 +51,27 @@ class ClientViewSet(viewsets.ModelViewSet): class EntryViewSet(viewsets.ModelViewSet): + """Lookup Entry when authenticated, and bind to the serializer.""" + queryset = Entry.objects.filter(is_active=True) permission_classes = (IsAuthenticated,) serializer_class = EntrySerializer lookup_value_regex = ".*" http_method_names = ["get", "post", "head", "delete"] - # Ovveride the permissions classes for POST method since we want to accept Entry creates from any client - # Note: We make authorization decisions on whether to actually create the object in the perform_create method later def get_permissions(self): + """ + Override the permissions classes for POST method since we want to accept Entry creates from any client. + + Note: We make authorization decisions on whether to actually create the object in the perform_create method + later. + """ if self.request.method == "POST": return [AllowAny()] return super().get_permissions() def perform_create(self, serializer): + """Create a new Entry, causing that route to receive the actiontype (i.e. block).""" actiontype = serializer.validated_data["actiontype"] route = serializer.validated_data["route"] if self.request.user.username: @@ -126,6 +141,7 @@ def perform_create(self, serializer): @staticmethod def find_entries(arg, active_filter=None): + """Query entries either by pk or overlapping route.""" if not arg: return Entry.objects.none() @@ -149,6 +165,7 @@ def find_entries(arg, active_filter=None): return Entry.objects.filter(query) def retrieve(self, request, pk=None, **kwargs): + """Limit retrieval to a single route.""" entries = self.find_entries(pk, active_filter=True) # TODO: What happens if we get multiple? Is that ok? I think yes, and return them all? if entries.count() != 1: @@ -157,6 +174,7 @@ def retrieve(self, request, pk=None, **kwargs): return Response(serializer.data) def destroy(self, request, pk=None, *args, **kwargs): + """Only delete active (e.g. announced) entries.""" for entry in self.find_entries(pk, active_filter=True): entry.delete() diff --git a/scram/route_manager/apps.py b/scram/route_manager/apps.py index 319a8d17..bf13bf9e 100644 --- a/scram/route_manager/apps.py +++ b/scram/route_manager/apps.py @@ -1,5 +1,9 @@ +"""Register ourselves with Django.""" + from django.apps import AppConfig class RouteManagerConfig(AppConfig): + """Define the name of the module that's the main app.""" + name = "scram.route_manager" diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index 1c549376..dc151718 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -1,12 +1,15 @@ +"""Define one or more custom auth backends.""" + from django.conf import settings from django.contrib.auth.models import Group from mozilla_django_oidc.auth import OIDCAuthenticationBackend class ESnetAuthBackend(OIDCAuthenticationBackend): - def update_groups(self, user, claims): - """Sets the users group(s) to whatever is in the claims.""" + """Extend the OIDC backend with a custom permission model.""" + def update_groups(self, user, claims): + """Set the user's group(s) to whatever is in the claims.""" claimed_groups = claims.get("groups", []) effective_groups = [] @@ -38,10 +41,12 @@ def update_groups(self, user, claims): user.save() def create_user(self, claims): + """Wrap the superclass's user creation.""" user = super(ESnetAuthBackend, self).create_user(claims) return self.update_user(user, claims) def update_user(self, user, claims): + """Determine the user name from the claims.""" user.name = claims.get("given_name", "") + " " + claims.get("family_name", "") user.username = claims.get("preferred_username", "") if claims.get("groups", False): diff --git a/scram/route_manager/context_processors.py b/scram/route_manager/context_processors.py index 3781c3d2..61abde75 100644 --- a/scram/route_manager/context_processors.py +++ b/scram/route_manager/context_processors.py @@ -1,8 +1,11 @@ +"""Define custom functions that take a request and add to the context before template rendering.""" + from django.conf import settings from django.urls import reverse def login_logout(request): + """Pass through the relevant URLs from the settings.""" login_url = reverse(settings.LOGIN_URL) logout_url = reverse(settings.LOGOUT_URL) return {"login": login_url, "logout": logout_url} diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index ef063ead..101c8bd7 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -1,3 +1,5 @@ +"""Define the models used in the route_manager app.""" + import logging import uuid as uuid_lib @@ -10,33 +12,36 @@ class Route(models.Model): - """Model describing a route""" + """Define a route as a CIDR route and a UUID.""" route = CidrAddressField(unique=True) uuid = models.UUIDField(db_index=True, default=uuid_lib.uuid4, editable=False) def get_absolute_url(self): + """Ensure we use UUID on the API side instead.""" return reverse("") def __str__(self): + """Don't display the UUID, only the route.""" return str(self.route) class ActionType(models.Model): - """Defines an action that can be done with a given route. e.g. Block, shunt, redirect, etc.""" + """Define a type of action that can be done with a given route. e.g. Block, shunt, redirect, etc.""" name = models.CharField(help_text="One-word description of the action", max_length=30) available = models.BooleanField(help_text="Is this a valid choice for new entries?", default=True) history = HistoricalRecords() def __str__(self): + """Display clearly whether or not the action is currently available.""" if not self.available: return f"{self.name} (Inactive)" return self.name class WebSocketMessage(models.Model): - """Defines a single message sent to downstream translators via WebSocket.""" + """Define a single message sent to downstream translators via WebSocket.""" msg_type = models.CharField("The type of the message", max_length=50) msg_data = models.JSONField("The JSON payload. See also msg_data_route_field.", default=dict) @@ -47,11 +52,12 @@ class WebSocketMessage(models.Model): ) def __str__(self): + """Display clearly what the fields are used for.""" return f"{self.msg_type}: {self.msg_data} with the route in key {self.msg_data_route_field}" class WebSocketSequenceElement(models.Model): - """In a sequence of messages, defines a single element.""" + """In a sequence of messages, define a single element.""" websocketmessage = models.ForeignKey("WebSocketMessage", on_delete=models.CASCADE) order_num = models.SmallIntegerField( @@ -70,6 +76,7 @@ class WebSocketSequenceElement(models.Model): action_type = models.ForeignKey("ActionType", on_delete=models.CASCADE) def __str__(self): + """Summarize the fields into something short and readable.""" return ( f"{self.websocketmessage} as order={self.order_num} for " + f"{self.verb} actions on actiontype={self.action_type}" @@ -96,6 +103,7 @@ class Entry(models.Model): ) def delete(self, *args, **kwargs): + """Set inactive instead of deleting, as we want to ensure a history of entries.""" if not self.is_active: # We've already expired this route, don't send another message return @@ -115,36 +123,43 @@ def delete(self, *args, **kwargs): ) class Meta: + """Ensure that multiple routes can be added as long as they have different action types.""" + unique_together = ["route", "actiontype"] verbose_name_plural = "Entries" def __str__(self): + """Summarize the most important fields to something easily readable.""" desc = f"{self.route} ({self.actiontype})" if not self.is_active: desc += " (inactive)" return desc def get_change_reason(self): + """Traverse come complex relationships to determine the most recent change reason.""" hist_mgr = getattr(self, self._meta.simple_history_manager_attribute) return hist_mgr.order_by("-history_date").first().history_change_reason class IgnoreEntry(models.Model): - """For cidrs you NEVER want to block ie don't shoot yourself in the foot list""" + """Define CIDRs you NEVER want to block (i.e. the "don't shoot yourself in the foot" list).""" route = CidrAddressField(unique=True) comment = models.CharField(max_length=100) history = HistoricalRecords() class Meta: + """Ensure the plural is grammatically correct.""" + verbose_name_plural = "Ignored Entries" def __str__(self): + """Only display the route.""" return str(self.route) class Client(models.Model): - """Any client that would like to hit the API to add entries (e.g. Zeek)""" + """Any client that would like to hit the API to add entries (e.g. Zeek).""" hostname = models.CharField(max_length=50, unique=True) uuid = models.UUIDField() @@ -153,6 +168,7 @@ class Client(models.Model): authorized_actiontypes = models.ManyToManyField(ActionType) def __str__(self): + """Only display the hostname.""" return str(self.hostname) diff --git a/scram/route_manager/tests/__init__.py b/scram/route_manager/tests/__init__.py index e69de29b..fb031a32 100644 --- a/scram/route_manager/tests/__init__.py +++ b/scram/route_manager/tests/__init__.py @@ -0,0 +1 @@ +"""Define tests executed by pytest.""" diff --git a/scram/route_manager/tests/acceptance/environment.py b/scram/route_manager/tests/acceptance/environment.py index 5bae6990..0d31198b 100644 --- a/scram/route_manager/tests/acceptance/environment.py +++ b/scram/route_manager/tests/acceptance/environment.py @@ -1,9 +1,12 @@ +"""Setup the environment for tests.""" + from rest_framework.test import APIClient from scram.users.tests.factories import UserFactory def django_ready(context): + """Create a user that tests can use for authenticated/unauthenticated testing.""" # Create a user UserFactory(username="user", password="password") diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index 1d502e96..3885e4bb 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -1,3 +1,5 @@ +"""Define steps used extensively by the Behave tests.""" + import datetime import time @@ -12,6 +14,7 @@ @given("a {name} actiontype is defined") def step_impl(context, name): + """Create an actiontype of that name.""" context.channel_layer = get_channel_layer() async_to_sync(context.channel_layer.group_send)( f"translator_{name}", {"type": "translator_remove_all", "message": {}} @@ -26,6 +29,7 @@ def step_impl(context, name): @given("a client with {name} authorization") def step_impl(context, name): + """Create a client and authorize it for that action type.""" at, created = ActionType.objects.get_or_create(name=name) authorized_client = Client.objects.create( hostname="authorized_client.es.net", @@ -37,6 +41,7 @@ def step_impl(context, name): @given("a client without {name} authorization") def step_impl(context, name): + """Create a client that has no authorized action types.""" unauthorized_client = Client.objects.create( hostname="unauthorized_client.es.net", uuid="91e134a5-77cf-4560-9797-6bbdbffde9f8", @@ -46,22 +51,26 @@ def step_impl(context, name): @when("we're logged in") def step_impl(context): + """Login.""" context.test.client.login(username="user", password="password") @when("the CIDR prefix limits are {v4_minprefix:d} and {v6_minprefix:d}") def step_impl(context, v4_minprefix, v6_minprefix): + """Override our settings with the provided values.""" conf.settings.V4_MINPREFIX = v4_minprefix conf.settings.V6_MINPREFIX = v6_minprefix @then("we get a {status_code:d} status code") def step_impl(context, status_code): + """Ensure the status code response matches the expected value.""" context.test.assertEqual(context.response.status_code, status_code) @when("we add the entry {value:S}") def step_impl(context, value): + """Block the provided route.""" context.response = context.test.client.post( reverse("api:v1:entry-list"), { @@ -78,6 +87,7 @@ def step_impl(context, value): @when("we add the entry {value:S} with comment {comment}") def step_impl(context, value, comment): + """Block the provided route and add a comment.""" context.response = context.test.client.post( reverse("api:v1:entry-list"), { @@ -93,6 +103,7 @@ def step_impl(context, value, comment): @when("we add the entry {value:S} with expiration {exp:S}") def step_impl(context, value, exp): + """Block the provided route and add an absolute expiration datetime.""" context.response = context.test.client.post( reverse("api:v1:entry-list"), { @@ -109,6 +120,7 @@ def step_impl(context, value, exp): @when("we add the entry {value:S} with expiration in {secs:d} seconds") def step_impl(context, value, secs): + """Block the provided route and add a relative expiration.""" td = datetime.timedelta(seconds=secs) expiration = datetime.datetime.now() + td @@ -128,16 +140,19 @@ def step_impl(context, value, secs): @step("we wait {secs:d} seconds") def step_impl(context, secs): + """Wait to allow messages to propagate.""" time.sleep(secs) @then("we remove expired entries") def step_impl(context): + """Call the function that removes expired entries.""" context.response = context.test.client.get(reverse("route_manager:process-expired")) @when("we add the ignore entry {value:S}") def step_impl(context, value): + """Add an IgnoreEntry with the specified route.""" context.response = context.test.client.post( reverse("api:v1:ignoreentry-list"), {"route": value, "comment": "test api"} ) @@ -145,16 +160,19 @@ def step_impl(context, value): @when("we remove the {model} {value}") def step_impl(context, model, value): + """Remove any model object with the matching value.""" context.response = context.test.client.delete(reverse(f"api:v1:{model.lower()}-detail", args=[value])) @when("we list the {model}s") def step_impl(context, model): + """List all objects of an arbitrary model.""" context.response = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) @when("we update the {model} {value_from} to {value_to}") def step_impl(context, model, value_from, value_to): + """Modify any model object with the matching value to the new value instead.""" context.response = context.test.client.patch( reverse(f"api:v1:{model.lower()}-detail", args=[value_from]), {model.lower(): value_to}, @@ -163,6 +181,7 @@ def step_impl(context, model, value_from, value_to): @then("the number of {model}s is {num:d}") def step_impl(context, model, num): + """Count the number of objects of an arbitrary model.""" objs = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) context.test.assertEqual(len(objs.json()), num) @@ -172,6 +191,7 @@ def step_impl(context, model, num): @then("{value} is one of our list of {model}s") def step_impl(context, value, model): + """Ensure that the arbitrary model has an object with the specified value.""" objs = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) found = False @@ -187,6 +207,7 @@ def step_impl(context, value, model): @when("we register a client named {hostname} with the uuid of {uuid}") def step_impl(context, hostname, uuid): + """Create a client with a specific UUID.""" context.response = context.test.client.post( reverse("api:v1:client-list"), { diff --git a/scram/route_manager/tests/acceptance/steps/ip.py b/scram/route_manager/tests/acceptance/steps/ip.py index a346e954..8154f04b 100644 --- a/scram/route_manager/tests/acceptance/steps/ip.py +++ b/scram/route_manager/tests/acceptance/steps/ip.py @@ -1,12 +1,14 @@ +"""Define steps used for IP-related logic by the Behave tests.""" + import ipaddress from behave import then, when from django.urls import reverse -# This does a CIDR match @then("{route} is contained in our list of {model}s") def step_impl(context, route, model): + """Perform a CIDR match on the matching object.""" objs = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) ip_target = ipaddress.ip_address(route) @@ -22,6 +24,7 @@ def step_impl(context, route, model): @when("we query for {ip}") def step_impl(context, ip): + """Find an Entry for the specified IP.""" try: context.response = context.test.client.get(reverse("api:v1:entry-detail", args=[ip])) context.queryException = None @@ -32,11 +35,13 @@ def step_impl(context, ip): @then("we get a ValueError") def step_impl(context): + """Ensure we received a ValueError exception.""" assert isinstance(context.queryException, ValueError) @then("the change entry for {value:S} is {comment}") def step_impl(context, value, comment): + """Verify the comment for the Entry.""" try: objs = context.test.client.get(reverse("api:v1:entry-detail", args=[value])) context.test.assertEqual(objs.json()[0]["comment"], comment) diff --git a/scram/route_manager/tests/acceptance/steps/translator.py b/scram/route_manager/tests/acceptance/steps/translator.py index 85621f21..c6c2ff49 100644 --- a/scram/route_manager/tests/acceptance/steps/translator.py +++ b/scram/route_manager/tests/acceptance/steps/translator.py @@ -1,3 +1,5 @@ +"""Define steps used for translator integration testing by the Behave tests.""" + from behave import then from behave.api.async_step import async_run_until_complete from channels.testing import WebsocketCommunicator @@ -6,6 +8,7 @@ async def query_translator(route, actiontype, is_announced=True): + """Ensure the specified route is currently either blocked or unblocked.""" communicator = WebsocketCommunicator(ws_application, f"/ws/route_manager/webui_{actiontype}/") connected, subprotocol = await communicator.connect() assert connected @@ -21,10 +24,12 @@ async def query_translator(route, actiontype, is_announced=True): @then("{route} is announced by {actiontype} translators") @async_run_until_complete async def step_impl(context, route, actiontype): + """Ensure the specified route is currently blocked.""" await query_translator(route, actiontype) @then("{route} is not announced by {actiontype} translators") @async_run_until_complete async def step_impl(context, route, actiontype): + """Ensure the specified route is currently unblocked.""" await query_translator(route, actiontype, is_announced=False) diff --git a/scram/route_manager/tests/functional_tests.py b/scram/route_manager/tests/functional_tests.py index c4312ecf..c4515b13 100644 --- a/scram/route_manager/tests/functional_tests.py +++ b/scram/route_manager/tests/functional_tests.py @@ -1,7 +1,10 @@ +"""Use the Django web client to perform end-to-end, WebUI-based testing.""" + import unittest class HomePageTest(unittest.TestCase): + """Ensure the home page works.""" pass diff --git a/scram/route_manager/tests/test_api.py b/scram/route_manager/tests/test_api.py index 8152b877..1a81e907 100644 --- a/scram/route_manager/tests/test_api.py +++ b/scram/route_manager/tests/test_api.py @@ -1,3 +1,5 @@ +"""Use pytest to unit test the API.""" + from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import status @@ -7,7 +9,10 @@ class TestAddRemoveIP(APITestCase): + """Ensure that we can block IPs, and that duplicate blocks don't generate an error.""" + def setUp(self): + """Set up the environment for our tests.""" self.url = reverse("api:v1:entry-list") self.superuser = get_user_model().objects.create_superuser("admin", "admin@es.net", "admintestpassword") self.client.login(username="admin", password="admintestpassword") @@ -19,6 +24,7 @@ def setUp(self): self.authorized_client.authorized_actiontypes.set([1]) def test_block_ipv4(self): + """Block a v4 IP.""" response = self.client.post( self.url, { @@ -31,6 +37,7 @@ def test_block_ipv4(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) def test_block_duplicate_ipv4(self): + """Block an existing v4 IP and ensure we don't get an error.""" self.client.post( self.url, { @@ -52,6 +59,7 @@ def test_block_duplicate_ipv4(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) def test_block_ipv6(self): + """Block a v6 IP.""" response = self.client.post( self.url, { @@ -64,6 +72,7 @@ def test_block_ipv6(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) def test_block_duplicate_ipv6(self): + """Block an existing v6 IP and ensure we don't get an error.""" self.client.post( self.url, { @@ -86,11 +95,15 @@ def test_block_duplicate_ipv6(self): class TestUnauthenticatedAccess(APITestCase): + """Ensure that an unathenticated client can't do anything.""" + def setUp(self): + """Define some helper variables.""" self.entry_url = reverse("api:v1:entry-list") self.ignore_url = reverse("api:v1:ignoreentry-list") def test_unauthenticated_users_have_no_create_access(self): + """Ensure an unauthenticated client can't add an Entry.""" response = self.client.post( self.entry_url, { @@ -104,9 +117,11 @@ def test_unauthenticated_users_have_no_create_access(self): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_unauthenticated_users_have_no_ignore_create_access(self): + """Ensure an unauthenticated client can't add an IgnoreEntry.""" response = self.client.post(self.ignore_url, {"route": "192.0.2.4"}, format="json") self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) def test_unauthenticated_users_have_no_list_access(self): + """Ensure an unauthenticated client can't list Entries.""" response = self.client.get(self.entry_url, format="json") self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) diff --git a/scram/route_manager/tests/test_authorization.py b/scram/route_manager/tests/test_authorization.py index 6f4269d1..54f372b0 100644 --- a/scram/route_manager/tests/test_authorization.py +++ b/scram/route_manager/tests/test_authorization.py @@ -1,3 +1,5 @@ +"""Define tests for authorization and permissions.""" + from django.conf import settings from django.contrib.auth.models import Group from django.test import Client, TestCase @@ -10,7 +12,10 @@ class AuthzTest(TestCase): + """Define tests using the built-in authentication.""" + def setUp(self): + """Define several users for our tests.""" self.client = Client() self.unauthorized_user = User.objects.create(username="unauthorized") @@ -49,6 +54,7 @@ def setUp(self): ) def create_entry(self): + """Ensure the admin user can create an Entry.""" self.client.force_login(self.admin_user) self.client.post( reverse("route_manager:add"), @@ -63,8 +69,7 @@ def create_entry(self): return Entry.objects.latest("id").id def test_unauthorized_add_entry(self): - """Unauthorized users should not be able to add an entry""" - + """Unauthorized users should not be able to add an Entry.""" for user in self.write_blocked_users: if user: self.client.force_login(user) @@ -79,6 +84,7 @@ def test_unauthorized_add_entry(self): self.assertEqual(response.status_code, 302) def test_authorized_add_entry(self): + """Test authorized users with various permissions to ensure they can add an Entry.""" for user in self.write_allowed_users: self.client.force_login(user) response = self.client.post( @@ -92,6 +98,7 @@ def test_authorized_add_entry(self): self.assertEqual(response.status_code, 200) def test_unauthorized_detail_view(self): + """Ensure that unauthorized users can't view the blocked IPs.""" pk = self.create_entry() for user in self.detail_blocked_users: @@ -101,6 +108,7 @@ def test_unauthorized_detail_view(self): self.assertIn(response.status_code, [302, 403], msg=f"username={user}") def test_authorized_detail_view(self): + """Test authorized users with various permissions to ensure they can view block details.""" pk = self.create_entry() for user in self.detail_allowed_users: @@ -110,7 +118,6 @@ def test_authorized_detail_view(self): def test_unauthorized_after_group_removal(self): """The user has r/w access, then when we remove them from the r/w group, they no longer do.""" - test_user = User.objects.create(username="tmp_readwrite") test_user.groups.set([self.readwrite_group]) test_user.save() @@ -126,7 +133,10 @@ def test_unauthorized_after_group_removal(self): class OidcTest(TestCase): + """Define tests using OIDC authentication.""" + def setUp(self): + """Create a sample OIDC user.""" self.client = Client() self.claims = { "given_name": "Edward", @@ -136,8 +146,7 @@ def setUp(self): } def test_unauthorized(self): - """A user with no groups should have no access""" - + """A user with no groups should have no access.""" claims = dict(self.claims) user = ESnetAuthBackend().create_user(claims) @@ -146,8 +155,7 @@ def test_unauthorized(self): self.assertEqual(list(user.user_permissions.all()), []) def test_readonly(self): - """Test r/o groups""" - + """Test r/o groups.""" claims = dict(self.claims) claims["groups"] = [settings.SCRAM_READONLY_GROUPS[0]] user = ESnetAuthBackend().create_user(claims) @@ -158,8 +166,7 @@ def test_readonly(self): self.assertFalse(user.has_perm("route_manager.add_entry")) def test_readwrite(self): - """Test r/w groups""" - + """Test r/w groups.""" claims = dict(self.claims) claims["groups"] = [settings.SCRAM_READWRITE_GROUPS[0]] user = ESnetAuthBackend().create_user(claims) @@ -171,8 +178,7 @@ def test_readwrite(self): self.assertTrue(user.has_perm("route_manager.add_entry")) def test_admin(self): - """Test admin_groups""" - + """Test admin_groups.""" claims = dict(self.claims) claims["groups"] = [settings.SCRAM_ADMIN_GROUPS[0]] user = ESnetAuthBackend().create_user(claims) @@ -183,8 +189,7 @@ def test_admin(self): self.assertTrue(user.has_perm("route_manager.add_entry")) def test_authorized_removal(self): - """Have an authorized user, then downgrade them and make sure they're unauthorized""" - + """Have an authorized user, then downgrade them and make sure they're unauthorized.""" claims = dict(self.claims) claims["groups"] = [settings.SCRAM_ADMIN_GROUPS[0]] user = ESnetAuthBackend().create_user(claims) @@ -230,7 +235,6 @@ def test_authorized_removal(self): def test_disabled(self): """Pass all the groups, user should be disabled as it takes precedence.""" - claims = dict(self.claims) claims["groups"] = [settings.SCRAM_GROUPS] user = ESnetAuthBackend().create_user(claims) diff --git a/scram/route_manager/tests/test_history.py b/scram/route_manager/tests/test_history.py index f00f16bb..b234e621 100644 --- a/scram/route_manager/tests/test_history.py +++ b/scram/route_manager/tests/test_history.py @@ -1,3 +1,5 @@ +"""Define tests for the history feature.""" + from django.test import TestCase from simple_history.utils import get_change_reason_from_object, update_change_reason @@ -5,10 +7,14 @@ class TestActiontypeHistory(TestCase): + """Test the history on an action type.""" + def setUp(self): + """Set up the test environment.""" self.atype = ActionType.objects.create(name="Block") def test_comments(self): + """Ensure we can go back and set a reason.""" self.atype.name = "Nullroute" self.atype._change_reason = "Use more descriptive name" self.atype.save() @@ -16,9 +22,12 @@ def test_comments(self): class TestEntryHistory(TestCase): + """Test the history on an Entry.""" + routes = ["192.0.2.16/32", "198.51.100.16/28"] def setUp(self): + """Set up the test environment.""" self.atype = ActionType.objects.create(name="Block") for r in self.routes: route = Route.objects.create(route=r) @@ -28,6 +37,7 @@ def setUp(self): self.assertEqual(entry.get_change_reason(), create_reason) def test_comments(self): + """Ensure we can update the reason.""" for r in self.routes: route_old = Route.objects.get(route=r) e = Entry.objects.get(route=route_old) diff --git a/scram/route_manager/tests/test_views.py b/scram/route_manager/tests/test_views.py index 415cdae0..bcc3a4ad 100644 --- a/scram/route_manager/tests/test_views.py +++ b/scram/route_manager/tests/test_views.py @@ -1,3 +1,5 @@ +"""Define simple tests for the template-based Views.""" + from django.test import TestCase from django.urls import resolve @@ -5,6 +7,9 @@ class HomePageTest(TestCase): + """Test how the home page renders.""" + def test_root_url_resolves_to_home_page_view(self): + """Ensure we can find the home page.""" found = resolve("/") self.assertEqual(found.func, home_page) diff --git a/scram/route_manager/tests/test_websockets.py b/scram/route_manager/tests/test_websockets.py index 367359b8..306e772b 100644 --- a/scram/route_manager/tests/test_websockets.py +++ b/scram/route_manager/tests/test_websockets.py @@ -1,3 +1,5 @@ +"""Define unit tests for the websockets-based communication.""" + import json from asyncio import gather from contextlib import asynccontextmanager @@ -15,7 +17,7 @@ @asynccontextmanager async def get_communicators(actiontypes, should_match, *args, **kwds): - """Creates a set of communicators, and then handles tear-down. + """Create a set of communicators, and then handle tear-down. Given two lists of the same length, a set of actiontypes, and set of boolean values, creates that many communicators, one for each actiontype-bool pair. @@ -48,6 +50,7 @@ class TestTranslatorBaseCase(TestCase): """Base case that other test cases build on top of. Three translators in one group, test one v4 and one v6.""" def setUp(self): + """Set up our test environment.""" # TODO: This is copied from test_api; should de-dupe this. self.url = reverse("api:v1:entry-list") self.superuser = get_user_model().objects.create_superuser("admin", "admin@example.net", "admintestpassword") @@ -80,7 +83,7 @@ def setUp(self): self.local_setUp() def local_setUp(self): - # Allow child classes to override this if desired + """Allow child classes to override this if desired.""" return async def get_messages(self, communicator, messages, should_match): @@ -95,6 +98,7 @@ async def get_nothings(self, communicator): assert await communicator.receive_nothing(timeout=0.1, interval=0.01) is False async def add_ip(self, ip, mask): + """Ensure we can add an IP to block.""" async with get_communicators(self.actiontypes, self.should_match) as communicators: await self.api_create_entry(ip) @@ -119,6 +123,7 @@ async def ensure_no_more_msgs(self, communicators): # Django ensures that the create is synchronous, so we have some extra steps to do @sync_to_async def api_create_entry(self, route): + """Ensure we can create an Entry via the API.""" return self.client.post( self.url, { @@ -131,12 +136,14 @@ def api_create_entry(self, route): ) async def test_add_v4(self): + """Test adding a few v4 routes.""" await self.add_ip("192.0.2.224", 32) await self.add_ip("192.0.2.225", 32) await self.add_ip("192.0.2.226", 32) await self.add_ip("198.51.100.224", 32) async def test_add_v6(self): + """Test adding a few v6 routes.""" await self.add_ip("2001:DB8:FDF0::", 128) await self.add_ip("2001:DB8:FDF0::D", 128) await self.add_ip("2001:DB8:FDF0::DB", 128) @@ -147,6 +154,7 @@ class TranslatorDontCrossTheStreamsTestCase(TestTranslatorBaseCase): """Two translators in the same group, two in another group, single IP, ensure we get only the messages we expect.""" def local_setUp(self): + """Define the actions and what we expect.""" self.actiontypes = ["block", "block", "noop", "noop"] self.should_match = [True, True, False, False] @@ -155,6 +163,7 @@ class TranslatorSequenceTestCase(TestTranslatorBaseCase): """Test a sequence of WebSocket messages.""" def local_setUp(self): + """Define the messages we want to send.""" wsm2 = WebSocketMessage.objects.create(msg_type="translator_add", msg_data_route_field="foo") _ = WebSocketSequenceElement.objects.create( websocketmessage=wsm2, verb="A", action_type=self.actiontype, order_num=20 @@ -175,6 +184,7 @@ class TranslatorParametersTestCase(TestTranslatorBaseCase): """Additional parameters in the JSONField.""" def local_setUp(self): + """Define the message we want to send.""" wsm = WebSocketMessage.objects.get(msg_type="translator_add", msg_data_route_field="route") wsm.msg_data = {"asn": 65550, "community": 100, "route": "Ensure this gets overwritten."} wsm.save() diff --git a/scram/route_manager/urls.py b/scram/route_manager/urls.py index dde55639..2735cc64 100644 --- a/scram/route_manager/urls.py +++ b/scram/route_manager/urls.py @@ -1,3 +1,5 @@ +"""Register URLs known to Django, and the View that will handle each.""" + from django.urls import path from . import views diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 54ff321a..99cc2c52 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -1,3 +1,5 @@ +"""Define the Views that will handle the HTTP requests.""" + import ipaddress import json @@ -23,6 +25,7 @@ def home_page(request, prefilter=Entry.objects.all()): + """Return the home page, autocreating a user if none exists.""" num_entries = settings.RECENT_LIMIT if request.user.has_perms(("route_manager.view_entry", "route_manager.add_entry")): readwrite = True @@ -60,6 +63,7 @@ def home_page(request, prefilter=Entry.objects.all()): def search_entries(request): + """Wrap the home page with a specified CIDR to restrict Entries to.""" # Using ipaddress because we needed to turn off strict mode # (which netfields uses by default with seemingly no toggle) # This caused searches with host bits set to 500 which is bad UX see: 68854ee1ad4789a62863083d521bddbc96ab7025 @@ -77,11 +81,14 @@ def search_entries(request): @require_POST @permission_required(["route_manager.view_entry", "route_manager.delete_entry"]) def delete_entry(request, pk): + """Wrap delete via the API and redirect to the home page.""" delete_entry_api(request, pk) return redirect("route_manager:home") class EntryDetailView(PermissionRequiredMixin, DetailView): + """Define a view for the API to use.""" + permission_required = ["route_manager.view_entry"] model = Entry template_name = "route_manager/entry_detail.html" @@ -92,6 +99,7 @@ class EntryDetailView(PermissionRequiredMixin, DetailView): @permission_required(["route_manager.view_entry", "route_manager.add_entry"]) def add_entry(request): + """Send a WebSocket message when adding a new entry.""" with transaction.atomic(): res = add_entry_api(request) @@ -121,6 +129,7 @@ def add_entry(request): def process_expired(request): + """For entries with an expiration, set them to inactive if expired. Return some simple stats.""" current_time = timezone.now() with transaction.atomic(): entries_start = Entry.objects.filter(is_active=True).count() @@ -140,10 +149,13 @@ def process_expired(request): class EntryListView(ListView): + """Define a view for the API to use.""" + model = Entry template_name = "route_manager/entry_list.html" def get_context_data(self, **kwargs): + """Group entries by action type.""" context = {"entries": {}} for at in ActionType.objects.all(): queryset = Entry.objects.filter(actiontype=at).order_by("-pk") diff --git a/scram/users/__init__.py b/scram/users/__init__.py index e69de29b..8728ced7 100644 --- a/scram/users/__init__.py +++ b/scram/users/__init__.py @@ -0,0 +1 @@ +"""The Users app defines users and a permission scheme for them.""" diff --git a/scram/users/admin.py b/scram/users/admin.py index c69eb541..ae35bc3f 100644 --- a/scram/users/admin.py +++ b/scram/users/admin.py @@ -1,3 +1,5 @@ +"""Register Admin models with the Django Admin site.""" + from django.contrib import admin from django.contrib.auth import admin as auth_admin from django.contrib.auth import get_user_model @@ -10,6 +12,7 @@ @admin.register(User) class UserAdmin(auth_admin.UserAdmin): + """Allow managing Users via the Django admin site.""" form = UserChangeForm add_form = UserCreationForm diff --git a/scram/users/api/serializers.py b/scram/users/api/serializers.py index c95bfe62..a49520d0 100644 --- a/scram/users/api/serializers.py +++ b/scram/users/api/serializers.py @@ -1,3 +1,5 @@ +"""Serializers provide mappings between the API and the underlying model.""" + from django.contrib.auth import get_user_model from rest_framework import serializers @@ -5,7 +7,11 @@ class UserSerializer(serializers.ModelSerializer): + """This serializer defines no new fields.""" + class Meta: + """Maps to the User model, and specifies the fields exposed by the API.""" + model = User fields = ["username", "name", "url"] diff --git a/scram/users/api/views.py b/scram/users/api/views.py index 288ea7ab..ce70a713 100644 --- a/scram/users/api/views.py +++ b/scram/users/api/views.py @@ -1,3 +1,5 @@ +"""Views provide mappings between the underlying model and how they're listed in the API.""" + from django.contrib.auth import get_user_model from rest_framework import status from rest_framework.decorators import action @@ -11,14 +13,18 @@ class UserViewSet(RetrieveModelMixin, ListModelMixin, UpdateModelMixin, GenericViewSet): + """Lookup Users by username.""" + serializer_class = UserSerializer queryset = User.objects.all() lookup_field = "username" def get_queryset(self, *args, **kwargs): + """Query on User ID.""" return self.queryset.filter(id=self.request.user.id) @action(detail=False, methods=["GET"]) def me(self, request): + """Return the current user.""" serializer = UserSerializer(request.user, context={"request": request}) return Response(status=status.HTTP_200_OK, data=serializer.data) diff --git a/scram/users/apps.py b/scram/users/apps.py index 2ce72578..449ebe39 100644 --- a/scram/users/apps.py +++ b/scram/users/apps.py @@ -1,3 +1,5 @@ +"""Register ourselves with Django.""" + import logging from django.apps import AppConfig @@ -7,10 +9,13 @@ class UsersConfig(AppConfig): + """Define the name of the module for the Users app.""" + name = "scram.users" verbose_name = _("Users") def ready(self): + """Check if signals are registered for User events.""" try: import scram.users.signals # noqa F401 except ImportError: diff --git a/scram/users/forms.py b/scram/users/forms.py index e2792fc2..b079f2a4 100644 --- a/scram/users/forms.py +++ b/scram/users/forms.py @@ -1,3 +1,5 @@ +"""Define forms for the Admin site.""" + from django.contrib.auth import forms as admin_forms from django.contrib.auth import get_user_model from django.utils.translation import gettext_lazy as _ @@ -6,12 +8,20 @@ class UserChangeForm(admin_forms.UserChangeForm): + """Define a form to edit a User.""" + class Meta(admin_forms.UserChangeForm.Meta): + """Map to the User model.""" + model = User class UserCreationForm(admin_forms.UserCreationForm): + """Define a form to create a User.""" + class Meta(admin_forms.UserCreationForm.Meta): + """Map to the User model and provide custom error messages.""" + model = User error_messages = {"username": {"unique": _("This username has already been taken.")}} diff --git a/scram/users/models.py b/scram/users/models.py index c2e1914f..81a905ca 100644 --- a/scram/users/models.py +++ b/scram/users/models.py @@ -1,3 +1,5 @@ +"""Define models for the User application.""" + from django.contrib.auth.models import AbstractUser from django.db.models import CharField from django.urls import reverse diff --git a/scram/users/tests/__init__.py b/scram/users/tests/__init__.py index e69de29b..f1101332 100644 --- a/scram/users/tests/__init__.py +++ b/scram/users/tests/__init__.py @@ -0,0 +1 @@ +"""Define tests for the User application.""" diff --git a/scram/users/tests/factories.py b/scram/users/tests/factories.py index edd306cb..1eca95c0 100644 --- a/scram/users/tests/factories.py +++ b/scram/users/tests/factories.py @@ -1,3 +1,5 @@ +"""Define Factory tests for the Users application.""" + from typing import Any, Sequence from django.contrib.auth import get_user_model @@ -6,6 +8,7 @@ class UserFactory(DjangoModelFactory): + """Test the UserFactory.""" username = Faker("user_name") email = Faker("email") @@ -13,6 +16,7 @@ class UserFactory(DjangoModelFactory): @post_generation def password(self, create: bool, extracted: Sequence[Any], **kwargs): + """Test password assignment.""" password = ( extracted if extracted @@ -28,5 +32,7 @@ def password(self, create: bool, extracted: Sequence[Any], **kwargs): self.set_password(password) class Meta: + """Map to User model.""" + model = get_user_model() django_get_or_create = ["username"] diff --git a/scram/users/tests/test_admin.py b/scram/users/tests/test_admin.py index 66f20cda..fd49e7ae 100644 --- a/scram/users/tests/test_admin.py +++ b/scram/users/tests/test_admin.py @@ -1,3 +1,5 @@ +"""Tests for the Admin site for the Users application.""" + import pytest from django.urls import reverse @@ -7,17 +9,22 @@ class TestUserAdmin: + """Test various User operations via the Admin site.""" + def test_changelist(self, admin_client): + """Ensure we can view the changelist.""" url = reverse("admin:users_user_changelist") response = admin_client.get(url) assert response.status_code == 200 def test_search(self, admin_client): + """Ensure we can view the search.""" url = reverse("admin:users_user_changelist") response = admin_client.get(url, data={"q": "test"}) assert response.status_code == 200 def test_add(self, admin_client): + """Ensure we can add a user.""" url = reverse("admin:users_user_add") response = admin_client.get(url) assert response.status_code == 200 @@ -34,6 +41,7 @@ def test_add(self, admin_client): assert User.objects.filter(username="test").exists() def test_view_user(self, admin_client): + """Ensure we can view a user.""" user = User.objects.get(username="admin") url = reverse("admin:users_user_change", kwargs={"object_id": user.pk}) response = admin_client.get(url) diff --git a/scram/users/tests/test_drf_urls.py b/scram/users/tests/test_drf_urls.py index 6e28c266..8e587486 100644 --- a/scram/users/tests/test_drf_urls.py +++ b/scram/users/tests/test_drf_urls.py @@ -1,3 +1,5 @@ +"""Test Django Request Framework use of the Users application.""" + import pytest from django.urls import resolve, reverse @@ -7,15 +9,18 @@ def test_user_detail(user: User): + """Ensure we can view details for a single User.""" assert reverse("api:v1:user-detail", kwargs={"username": user.username}) == f"/api/v1/users/{user.username}/" assert resolve(f"/api/v1/users/{user.username}/").view_name == "api:v1:user-detail" def test_user_list(): + """Ensure we can list all Users.""" assert reverse("api:v1:user-list") == "/api/v1/users/" assert resolve("/api/v1/users/").view_name == "api:v1:user-list" def test_user_me(): + """Ensure we can view info for the current user.""" assert reverse("api:v1:user-me") == "/api/v1/users/me/" assert resolve("/api/v1/users/me/").view_name == "api:v1:user-me" diff --git a/scram/users/tests/test_drf_views.py b/scram/users/tests/test_drf_views.py index 6e7b19e5..bbe56598 100644 --- a/scram/users/tests/test_drf_views.py +++ b/scram/users/tests/test_drf_views.py @@ -1,3 +1,5 @@ +"""Test Django Request Framework Views in the Users application.""" + import pytest from django.test import RequestFactory @@ -8,7 +10,10 @@ class TestUserViewSet: + """Test a couple simple View operations.""" + def test_get_queryset(self, user: User, rf: RequestFactory): + """Ensure we can view an arbitrary URL.""" view = UserViewSet() request = rf.get("/fake-url/") request.user = user @@ -18,6 +23,7 @@ def test_get_queryset(self, user: User, rf: RequestFactory): assert user in view.get_queryset() def test_me(self, user: User, rf: RequestFactory): + """Ensure we can view info on the current user.""" view = UserViewSet() request = rf.get("/fake-url/") request.user = user diff --git a/scram/users/tests/test_forms.py b/scram/users/tests/test_forms.py index 6d0b248b..4ff89f37 100644 --- a/scram/users/tests/test_forms.py +++ b/scram/users/tests/test_forms.py @@ -1,6 +1,5 @@ -""" -Module for all Form Tests. -""" +"""Module for all Form Tests.""" + import pytest from django.utils.translation import gettext_lazy as _ @@ -11,18 +10,16 @@ class TestUserCreationForm: - """ - Test class for all tests related to the UserCreationForm - """ + """Test class for all tests related to the UserCreationForm.""" def test_username_validation_error_msg(self, user: User): """ - Tests UserCreation Form's unique validator functions correctly by testing: + Tests UserCreation Form's unique validator functions correctly by testing 3 things. + 1) A new user with an existing username cannot be added. 2) Only 1 error is raised by the UserCreation Form 3) The desired error message is raised """ - # The user already exists, # hence cannot be created. form = UserCreationForm( diff --git a/scram/users/tests/test_models.py b/scram/users/tests/test_models.py index 9920c0ff..a224cd7d 100644 --- a/scram/users/tests/test_models.py +++ b/scram/users/tests/test_models.py @@ -1,3 +1,5 @@ +"""Define tests for the Models in the Users application.""" + import pytest from scram.users.models import User @@ -6,4 +8,5 @@ def test_user_get_absolute_url(user: User): + """Ensure w ecan convert a User object into a URL with info about that user.""" assert user.get_absolute_url() == f"/users/{user.username}/" diff --git a/scram/users/tests/test_urls.py b/scram/users/tests/test_urls.py index dd36a987..aa5a57ad 100644 --- a/scram/users/tests/test_urls.py +++ b/scram/users/tests/test_urls.py @@ -1,3 +1,5 @@ +"""Define tests for the URL resolution for the Users application.""" + import pytest from django.urls import resolve, reverse @@ -7,15 +9,18 @@ def test_detail(user: User): + """Ensure we can get the URL to view details about a single User.""" assert reverse("users:detail", kwargs={"username": user.username}) == f"/users/{user.username}/" assert resolve(f"/users/{user.username}/").view_name == "users:detail" def test_update(): + """Ensure we can get the URL to update a User.""" assert reverse("users:update") == "/users/~update/" assert resolve("/users/~update/").view_name == "users:update" def test_redirect(): + """Ensure we can get the URL to redirect to a User.""" assert reverse("users:redirect") == "/users/~redirect/" assert resolve("/users/~redirect/").view_name == "users:redirect" diff --git a/scram/users/tests/test_views.py b/scram/users/tests/test_views.py index 485370d2..63ad97bc 100644 --- a/scram/users/tests/test_views.py +++ b/scram/users/tests/test_views.py @@ -1,3 +1,5 @@ +"""Define tests for the Views of the Users application.""" + import pytest from django.conf import settings from django.contrib import messages @@ -17,6 +19,8 @@ class TestUserUpdateView: """ + Define tests related to the Update View. + TODO: extracting view initialization code as class-scoped fixture would be great if only pytest-django supported non-function-scoped @@ -25,6 +29,7 @@ class TestUserUpdateView: """ def test_get_success_url(self, user: User, rf: RequestFactory): + """Ensure we can view an arbitrary URL.""" view = UserUpdateView() request = rf.get("/fake-url/") request.user = user @@ -34,6 +39,7 @@ def test_get_success_url(self, user: User, rf: RequestFactory): assert view.get_success_url() == f"/users/{user.username}/" def test_get_object(self, user: User, rf: RequestFactory): + """Ensure we can retrieve the User object.""" view = UserUpdateView() request = rf.get("/fake-url/") request.user = user @@ -43,6 +49,7 @@ def test_get_object(self, user: User, rf: RequestFactory): assert view.get_object() == user def test_form_valid(self, user: User, rf: RequestFactory): + """Ensure the form validation works.""" view = UserUpdateView() request = rf.get("/fake-url/") @@ -63,7 +70,10 @@ def test_form_valid(self, user: User, rf: RequestFactory): class TestUserRedirectView: + """Define tests related to the Redirect View.""" + def test_get_redirect_url(self, user: User, rf: RequestFactory): + """Ensure the redirection URL works.""" view = UserRedirectView() request = rf.get("/fake-url") request.user = user @@ -74,7 +84,10 @@ def test_get_redirect_url(self, user: User, rf: RequestFactory): class TestUserDetailView: + """Define tests related to the User Detail View.""" + def test_authenticated(self, user: User, rf: RequestFactory): + """Ensure an authenticated user has access.""" request = rf.get("/fake-url/") request.user = UserFactory() @@ -83,6 +96,7 @@ def test_authenticated(self, user: User, rf: RequestFactory): assert response.status_code == 200 def test_not_authenticated(self, user: User, rf: RequestFactory): + """Ensure an unauthenticated user gets redirected to login.""" request = rf.get("/fake-url/") request.user = AnonymousUser() diff --git a/scram/users/urls.py b/scram/users/urls.py index 64f9d9cb..cf8647f3 100644 --- a/scram/users/urls.py +++ b/scram/users/urls.py @@ -1,3 +1,5 @@ +"""Register URLs known to Django, and the View that will handle each.""" + from django.urls import path from scram.users.views import user_detail_view, user_redirect_view, user_update_view diff --git a/scram/users/views.py b/scram/users/views.py index 2b077d42..e05f5cec 100644 --- a/scram/users/views.py +++ b/scram/users/views.py @@ -1,3 +1,5 @@ +"""Define the Views that will handle the HTTP requests.""" + from django.contrib.auth import get_user_model from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.messages.views import SuccessMessageMixin @@ -9,6 +11,7 @@ class UserDetailView(LoginRequiredMixin, DetailView): + """Map this view to the User model, and rely on the DetailView generic view.""" model = User slug_field = "username" @@ -19,15 +22,18 @@ class UserDetailView(LoginRequiredMixin, DetailView): class UserUpdateView(LoginRequiredMixin, SuccessMessageMixin, UpdateView): + """Map this view to the User model, and rely on the UpdateView generic view.""" model = User fields = ["name"] success_message = _("Information successfully updated") def get_success_url(self): + """Return the User detail view.""" return reverse("users:detail", kwargs={"username": self.request.user.username}) def get_object(self): + """Return the User object.""" return self.request.user @@ -35,10 +41,12 @@ def get_object(self): class UserRedirectView(LoginRequiredMixin, RedirectView): + """Map this view to the User model, and rely on the RedirectView generic view.""" permanent = False def get_redirect_url(self): + """Return the User detail view.""" return reverse("users:detail", kwargs={"username": self.request.user.username}) diff --git a/scram/utils/__init__.py b/scram/utils/__init__.py index e69de29b..e5a29f6c 100644 --- a/scram/utils/__init__.py +++ b/scram/utils/__init__.py @@ -0,0 +1 @@ +"""A module for defining utility functions and code.""" diff --git a/scram/utils/context_processors.py b/scram/utils/context_processors.py index 3c535141..2a86dfa4 100644 --- a/scram/utils/context_processors.py +++ b/scram/utils/context_processors.py @@ -1,8 +1,10 @@ +"""Define custom functions that take a request and add to the context before template rendering.""" + from django.conf import settings def settings_context(_request): - """Settings available by default to the templates context.""" + """Define settings available by default to the templates context.""" # Note: we intentionally do NOT expose the entire settings # to prevent accidental leaking of sensitive information return {"DEBUG": settings.DEBUG} From 954861020e2c067f64ad84c1609440049fd5e6aa Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 15:36:29 -0600 Subject: [PATCH 051/156] Require docstrings on everything on scram/ --- .github/workflows/flake8.yml | 20 ++++++++++++++++++++ requirements/local.txt | 1 + 2 files changed, 21 insertions(+) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 7f8d5c53..41626baf 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -31,3 +31,23 @@ jobs: - name: Run flake8 run: | flake8 + + # This will merge into the above once the other parts pass this check + ensure_docstrings: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache Docker images. + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }} + + - name: Install dependencies + run: | + pip install flake8 flake8-docstrings + + - name: Ensure scram has docstrings + run: | + flake8 scram diff --git a/requirements/local.txt b/requirements/local.txt index a0fbe77b..68accd63 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -18,6 +18,7 @@ sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild # Code quality # ------------------------------------------------------------------------------ +flake8-docstrings==1.7.0 # https://github.com/pycqa/flake8-docstrings flake8-isort==4.1.1 # https://github.com/gforcada/flake8-isort black==22.3.0 # https://github.com/psf/black pylint-django==2.5.3 # https://github.com/PyCQA/pylint-django From a969cc8b296d204910c8a0559c060ac7874e1b77 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 15:36:32 -0600 Subject: [PATCH 052/156] docs(envvars): Document all our envvars This is the ones we use in our ansible role; we should still do a better job documenting the required ones with their defaults --- docs/environment_variables.md | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/environment_variables.md diff --git a/docs/environment_variables.md b/docs/environment_variables.md new file mode 100644 index 00000000..c44629a7 --- /dev/null +++ b/docs/environment_variables.md @@ -0,0 +1,76 @@ +## Environment Variables to Set for Deployment +[comment]: # Which branch of SCRAM to use (you probably want to set it to a release tag) +scram_code_branch: +#### Systems +[comment]: # Email of the main admin +scram_manager_email: +[comment]: # Set to true for production mode; set to false to set up the compose.override.local.yml stack +scram_prod: true +[comment]: # Set to true if you want ansible to install a scram user +scram_install_user: true +[comment]: # What group to put `scram` user in +scram_group: 'scram' +[comment]: # What username to use for `scram` user +scram_user: '' +[comment]: # WHat uid to use for `scram` user +scram_uid: '' +[comment]: # What directory to use for base of the repo +scram_home: '/usr/local/scram' +[comment]: # IP or DNS record for your postgres host +scram_postgres_host: +[comment]: # What postgres user to use +scram_postgres_user: '' + +#### Authentication +[comment]: # This chooses if you want to use oidc or local accounts. This can be local or oidc only. Default: `local` +scram_auth_method: "local" +[comment]: # This client id (username) for your oidc connection. Only need to set this if you are trying to do oidc. +scram_oidc_client_id: + +#### Networking +[comment]: # What is the peering interface docker uses for gobgp to talk to the router +scram_peering_iface: 'ens192' +[comment]: # The v6 network of your peering connection +scram_v4_subnet: '10.0.0.0/24' +[comment]: # The v4 IP of the peering connection for the router side +scram_v4_gateway: '10.0.0.1' +[comment]: # The v4 IP of the peering connection for gobgp side +scram_v4_address: '10.0.0.2' +[comment]: # The v6 network of your peering connection +scram_v6_subnet: '2001:db8::/64' +[comment]: # The v6 IP of the peering connection for the router side +scram_v6_gateway: '2001:db8::2' +[comment]: # The v6 IP of the peering connection for the gobgp side +scram_v6_address: '2001:db8::3' +[comment]: # The AS you want to use for gobgp +scram_as: +[comment]: # A string representing your gobgp instance. Often seen as the local IP of the gobgp instance +scram_router_id: +[comment]: # +scram_peer_as: +[comment]: # The AS you want to use for gobgp side (can this be the same as `scram_as`?) +scram_local_as: +[comment]: # The fqdn of the server hosting this - to be used for nginx +scram_nginx_host: +[comment]: # List of allowed hosts per the django setting "ALLOWED_HOSTS". This should be a list of strings in shell +[comment]: # `django` is required for the websockets to work +[comment]: # Our Ansible assumes `django` + `scram_nginx_host` +scram_django_allowed_hosts: "django" +[comment]: # The fqdn of the server hosting this - to be used for nginx +scram_server_alias: +[comment]: # Do you want to set an md5 for authentication of bgp +scram_bgp_md5_enabled: false +[comment]: # The neighbor config of your gobgp config +scram_neighbors: +[comment]: # The v6 address of your neighbor + - neighbor_address: 2001:db8::2 +[comment]: # This is a v6 address so don't use v4 + ipv4: false +[comment]: # This is a v6 address so use v6 + ipv6: true +[comment]: # The v4 address of your neighbor + - neighbor_address: 10.0.0.200 +[comment]: # This is a v4 address so use v4 + ipv4: true +[comment]: # This is a v4 address so don't use v6 + ipv6: false From 7ca1132942b43a418a59058ca62829a191c6c889 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 15:39:45 -0600 Subject: [PATCH 053/156] docs(installation): add beginnings of environment variable instructions --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 9be49463..6e488bff 100644 --- a/README.rst +++ b/README.rst @@ -48,6 +48,9 @@ To get a basic implementation up and running locally: - Create ``$scram_home/.envs/.production/.postgres`` a template exists in the docs/templates directory - Make sure to set the right credentials - By default this template assumes you have a service defined in docker compose file called postgres. If you use another postgres server, make sure to update that setting as well +- Create a ``.env`` file with the necessary environment variables: + - [comment]: # This chooses if you want to use oidc or local accounts. This can be local or oidc only. Default: `local` + - scram_auth_method: "local" - ``make build`` - ``make toggle-prod`` - This will turn off debug mode in django and start using nginx to reverse proxy for the app From ef42a0ff8684ca6198cd7b112a4836541c7d446c Mon Sep 17 00:00:00 2001 From: chriscummings Date: Sun, 17 Nov 2024 15:46:54 -0600 Subject: [PATCH 054/156] chore(deps): bump django prod, whoops --- compose/production/django/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index e4a786a2..cbb5aeb5 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -1,5 +1,5 @@ -FROM python:3.8-slim-buster +FROM python:3.12-slim-bookwork ENV PYTHONUNBUFFERED 1 From bf023485be6f9bd03e5f1b1f4b67ec4704981323 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 15:59:15 -0600 Subject: [PATCH 055/156] Add docstrings to translator/ --- translator/exceptions.py | 8 ++------ translator/gobgp.py | 10 ++++++++++ translator/shared.py | 4 +--- translator/tests/acceptance/environment.py | 3 +++ .../acceptance/features/00_clean_slate.feature | 9 +++++++++ translator/tests/acceptance/steps/actions.py | 8 ++++++++ translator/translator.py | 14 +++++++++++--- 7 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 translator/tests/acceptance/features/00_clean_slate.feature diff --git a/translator/exceptions.py b/translator/exceptions.py index f1a54299..a999ec8c 100644 --- a/translator/exceptions.py +++ b/translator/exceptions.py @@ -1,9 +1,5 @@ -""" -This module holds all of the exceptions we want to raise in our translators. -""" +"""This module holds all of the exceptions we want to raise in our translators.""" class ASNError(TypeError): - """ - ASNError provides an error class to use when there is an issue with an Autonomous System Number. - """ + """ASNError provides an error class to use when there is an issue with an Autonomous System Number.""" diff --git a/translator/gobgp.py b/translator/gobgp.py index 23ae2572..6c10e159 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -1,3 +1,5 @@ +"""A translator interface for GoBGP (https://github.com/osrg/gobgp).""" + import logging import attribute_pb2 @@ -18,7 +20,10 @@ class GoBGP(object): + """Represents a GoBGP instance.""" + def __init__(self, url): + """Configure the channel used for communication.""" channel = grpc.insecure_channel(url) self.stub = gobgp_pb2_grpc.GobgpApiStub(channel) @@ -116,6 +121,7 @@ def _build_path(self, ip, event_data={}): ) def add_path(self, ip, event_data): + """Announce a single route.""" logging.info(f"Blocking {ip}") try: path = self._build_path(ip, event_data) @@ -128,11 +134,13 @@ def add_path(self, ip, event_data): logging.warning(f"ASN assertion failed with error: {e}") def del_all_paths(self): + """Remove all routes from being announced.""" logging.warning("Withdrawing ALL routes") self.stub.DeletePath(gobgp_pb2.DeletePathRequest(table_type=gobgp_pb2.GLOBAL), _TIMEOUT_SECONDS) def del_path(self, ip, event_data): + """Remove a single route from being announced.""" logging.info(f"Unblocking {ip}") try: path = self._build_path(ip, event_data) @@ -144,6 +152,7 @@ def del_path(self, ip, event_data): logging.warning(f"ASN assertion failed with error: {e}") def get_prefixes(self, ip): + """Retrieve the routes that match a prefix and are announced.""" prefixes = [gobgp_pb2.TableLookupPrefix(prefix=str(ip.ip))] family_afi = self._get_family_AFI(ip.ip.version) result = self.stub.ListPath( @@ -157,4 +166,5 @@ def get_prefixes(self, ip): return list(result) def is_blocked(self, ip): + """Return True if at least one route matching the prefix is being announced.""" return len(self.get_prefixes(ip)) > 0 diff --git a/translator/shared.py b/translator/shared.py index 1fe048bd..27e89f97 100644 --- a/translator/shared.py +++ b/translator/shared.py @@ -1,6 +1,4 @@ -""" -This module provides a location for code that we want to share between all translators. -""" +"""This module provides a location for code that we want to share between all translators.""" from exceptions import ASNError diff --git a/translator/tests/acceptance/environment.py b/translator/tests/acceptance/environment.py index 1f06b08d..b5c57b41 100644 --- a/translator/tests/acceptance/environment.py +++ b/translator/tests/acceptance/environment.py @@ -1,6 +1,9 @@ +"""Configure the test environment before executing acceptance tests.""" + from gobgp import GoBGP def before_all(context): + """Create a GoBGP object.""" context.gobgp = GoBGP("gobgp:50051") context.config.setup_logging() diff --git a/translator/tests/acceptance/features/00_clean_slate.feature b/translator/tests/acceptance/features/00_clean_slate.feature new file mode 100644 index 00000000..e4a9f47a --- /dev/null +++ b/translator/tests/acceptance/features/00_clean_slate.feature @@ -0,0 +1,9 @@ +Feature: Tests run correctly + When running tests, there are no leftover blocks + + Scenario: No leftover v4 blocks + Then 0.0.0.0/0 is unblocked + + Scenario: No leftover v6 blocks + Then ::/0 is unblocked + aaaaaaaaaaaaaaaa \ No newline at end of file diff --git a/translator/tests/acceptance/steps/actions.py b/translator/tests/acceptance/steps/actions.py index ce340de1..1e03e8e1 100644 --- a/translator/tests/acceptance/steps/actions.py +++ b/translator/tests/acceptance/steps/actions.py @@ -1,3 +1,5 @@ +"""Define the steps used by Behave.""" + import ipaddress import logging import time @@ -10,6 +12,7 @@ @when("we add {route} with {asn} and {community} to the block list") def add_block(context, route, asn, community): + """Block a single IP.""" ip = ipaddress.ip_interface(route) event_data = {"asn": int(asn), "community": int(community)} context.gobgp.add_path(ip, event_data) @@ -17,12 +20,14 @@ def add_block(context, route, asn, community): @then("we delete {route} with {asn} and {community} from the block list") def del_block(context, route, asn, community): + """Remove a single IP.""" ip = ipaddress.ip_interface(route) event_data = {"asn": int(asn), "community": int(community)} context.gobgp.del_path(ip, event_data) def get_block_status(context, ip): + """Check if the IP is currently blocked.""" # Allow our add/delete requests to settle time.sleep(1) @@ -38,15 +43,18 @@ def get_block_status(context, ip): @capture @when("{route} and {community} with invalid {asn} is sent") def asn_validation_fails(context, route, asn, community): + """Ensure the ASN was invalid.""" add_block(context, route, asn, community) assert context.log_capture.find_event("ASN assertion failed") @then("{ip} is blocked") def check_block(context, ip): + """Ensure that the IP is currently blocked.""" assert get_block_status(context, ip) @then("{ip} is unblocked") def check_unblock(context, ip): + """Ensure that the IP is currently unblocked.""" assert not get_block_status(context, ip) diff --git a/translator/translator.py b/translator/translator.py index 8521d7ab..b5a7dd02 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +"""Define the main event loop for the translator.""" + import asyncio import ipaddress import json @@ -14,9 +16,14 @@ if debug_mode: def install_deps(): - # Because of how we build translator currently, we don't have a great way to selectively install things at - # build, so we just do it here! Right now this also includes base.txt, which is unecessary, but in the - # future when we build a little better, it'll already be setup. + """ + Install necessary dependencies for debuggers. + + Because of how we build translator currently, we don't have a great way to selectively + install things at build, so we just do it here! Right now this also includes base.txt, + which is unecessary, but in the future when we build a little better, it'll already be + setup. + """ logging.info("Installing dependencies for debuggers") import subprocess @@ -58,6 +65,7 @@ def install_deps(): async def main(): + """Connect to the websocket and start listening for messages.""" g = GoBGP("gobgp:50051") async for websocket in websockets.connect(url): try: From 27293023eccd6e76d52f3e8f4ccda6629d5f4727 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 16:00:01 -0600 Subject: [PATCH 056/156] Enforce docstrings on translator too. --- .github/workflows/flake8.yml | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 41626baf..887ebc50 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -14,26 +14,6 @@ on: jobs: flake8: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache Docker images. - uses: ScribeMD/docker-cache@0.3.7 - with: - key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }} - - - name: Install dependencies - run: | - pip install flake8 - - - name: Run flake8 - run: | - flake8 - - # This will merge into the above once the other parts pass this check - ensure_docstrings: runs-on: ubuntu-latest steps: - name: Checkout code @@ -48,6 +28,6 @@ jobs: run: | pip install flake8 flake8-docstrings - - name: Ensure scram has docstrings + - name: Run flake8 run: | - flake8 scram + flake8 scram translator From 2e52f9fc8d3dc489054c2b27ffd41704531cef64 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 17 Nov 2024 16:06:10 -0600 Subject: [PATCH 057/156] I didn't actually mean to commit this, but feels small enough to leave for now. --- .../features/{00_clean_slate.feature => clean_slate.feature} | 1 - 1 file changed, 1 deletion(-) rename translator/tests/acceptance/features/{00_clean_slate.feature => clean_slate.feature} (91%) diff --git a/translator/tests/acceptance/features/00_clean_slate.feature b/translator/tests/acceptance/features/clean_slate.feature similarity index 91% rename from translator/tests/acceptance/features/00_clean_slate.feature rename to translator/tests/acceptance/features/clean_slate.feature index e4a9f47a..9de4e337 100644 --- a/translator/tests/acceptance/features/00_clean_slate.feature +++ b/translator/tests/acceptance/features/clean_slate.feature @@ -6,4 +6,3 @@ Feature: Tests run correctly Scenario: No leftover v6 blocks Then ::/0 is unblocked - aaaaaaaaaaaaaaaa \ No newline at end of file From 100bedfdfefad23cad400df4d19f337cec62ef82 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 22:30:12 -0600 Subject: [PATCH 058/156] docs(docstrings): update a few docstrings to better match the overall goal of a function or method --- scram/route_manager/api/exceptions.py | 2 +- scram/route_manager/api/views.py | 2 +- scram/route_manager/authentication_backends.py | 2 +- scram/route_manager/models.py | 2 +- scram/route_manager/tests/acceptance/steps/common.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scram/route_manager/api/exceptions.py b/scram/route_manager/api/exceptions.py index e75c87b9..74d6f2e2 100644 --- a/scram/route_manager/api/exceptions.py +++ b/scram/route_manager/api/exceptions.py @@ -5,7 +5,7 @@ class PrefixTooLarge(APIException): - """The CIDR prefix that was specified is larger than 32 bits for IPv4 of 128 bits for IPv6.""" + """The CIDR prefix that was specified is larger than the prefix allowed in the settings.""" v4_min_prefix = getattr(settings, "V4_MINPREFIX", 0) v6_min_prefix = getattr(settings, "V6_MINPREFIX", 0) diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index ca3a5d09..ad26127e 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -165,7 +165,7 @@ def find_entries(arg, active_filter=None): return Entry.objects.filter(query) def retrieve(self, request, pk=None, **kwargs): - """Limit retrieval to a single route.""" + """Retrieve a single route.""" entries = self.find_entries(pk, active_filter=True) # TODO: What happens if we get multiple? Is that ok? I think yes, and return them all? if entries.count() != 1: diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index dc151718..30288520 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -46,7 +46,7 @@ def create_user(self, claims): return self.update_user(user, claims) def update_user(self, user, claims): - """Determine the user name from the claims.""" + """Determine the user name from the claims and update said user's groups.""" user.name = claims.get("given_name", "") + " " + claims.get("family_name", "") user.username = claims.get("preferred_username", "") if claims.get("groups", False): diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index 101c8bd7..3bbf67e3 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -34,7 +34,7 @@ class ActionType(models.Model): history = HistoricalRecords() def __str__(self): - """Display clearly whether or not the action is currently available.""" + """Display clearly whether the action is currently available.""" if not self.available: return f"{self.name} (Inactive)" return self.name diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index 3885e4bb..c7c03f8c 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -1,4 +1,4 @@ -"""Define steps used extensively by the Behave tests.""" +"""Define steps used exclusively by the Behave tests.""" import datetime import time From 53153c1e8f8c597a2523ff9a48b8e5f7af7794ae Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 22:47:10 -0600 Subject: [PATCH 059/156] refactor(authentication-settings): move settings back into base and just override in local.py it was redundant between production and test. this does mean that you have to set the auth_method envvar even if you are using the local stack, but that's fine. --- config/settings/base.py | 61 +++++++++++++++++++++++++++++++ config/settings/local.py | 8 ++-- config/settings/production.py | 63 +------------------------------- config/settings/test.py | 69 +---------------------------------- 4 files changed, 68 insertions(+), 133 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 26ed8ec8..20e9e558 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -2,6 +2,7 @@ Base settings to build other settings files upon. """ +import logging import os from pathlib import Path @@ -279,6 +280,66 @@ # Are you using local passwords or oidc? AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() +logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") +if AUTH_METHOD == "oidc": + # Extend middleware to add OIDC middleware + MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 + + # Extend middleware to add OIDC auth backend + AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url + LOGIN_URL = "oidc_authentication_init" + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url + LOGIN_REDIRECT_URL = "route_manager:home" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url + LOGOUT_URL = "oidc_logout" + + # Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s + # https://github.com/mozilla/mozilla-django-oidc/issues/118 + # Using `/` because named urls don't work for this package + # https://github.com/mozilla/mozilla-django-oidc/issues/434 + LOGOUT_REDIRECT_URL = "route_manager:home" + + OIDC_OP_JWKS_ENDPOINT = os.environ.get( + "OIDC_OP_JWKS_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/certs", + ) + OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( + "OIDC_OP_AUTHORIZATION_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/auth", + ) + OIDC_OP_TOKEN_ENDPOINT = os.environ.get( + "OIDC_OP_TOKEN_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/token", + ) + OIDC_OP_USER_ENDPOINT = os.environ.get( + "OIDC_OP_USER_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", + ) + OIDC_RP_SIGN_ALGO = "RS256" + + OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") + OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") + +elif AUTH_METHOD == "local": + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url + LOGIN_URL = "local_auth:login" + + # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url + LOGIN_REDIRECT_URL = "route_manager:home" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url + LOGOUT_URL = "local_auth:logout" + + # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url + LOGOUT_REDIRECT_URL = "route_manager:home" +else: + raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") + + # Should we create an admin user for you AUTOCREATE_ADMIN = True diff --git a/config/settings/local.py b/config/settings/local.py index 9a6fd4de..74d6d0e0 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -1,5 +1,5 @@ from .base import * # noqa -from .base import env +from .base import AUTH_METHOD, env # GENERAL # ------------------------------------------------------------------------------ @@ -70,8 +70,10 @@ # AUTHENTICATION # ------------------------------------------------------------------------------ -# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url -LOGIN_REDIRECT_URL = "route_manager:home" +# We shouldn't be using OIDC in local dev mode as of now, but might be worth pursuing later +if AUTH_METHOD == "oidc": + raise NotImplementedError("oidc is not yet implemented") + # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "admin:login" # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url diff --git a/config/settings/production.py b/config/settings/production.py index 5e45125c..8f62b5d4 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -1,8 +1,5 @@ -import logging -import os - from .base import * # noqa -from .base import AUTH_METHOD, AUTHENTICATION_BACKENDS, MIDDLEWARE, env +from .base import env # GENERAL # ------------------------------------------------------------------------------ @@ -142,61 +139,3 @@ # Your stuff... # ------------------------------------------------------------------------------ -logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") -if AUTH_METHOD == "oidc": - # Extend middleware to add OIDC middleware - MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 - - # Extend middleware to add OIDC auth backend - AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 - - # https://docs.djangoproject.com/en/dev/ref/settings/#login-url - LOGIN_URL = "oidc_authentication_init" - - # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "route_manager:home" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url - LOGOUT_URL = "oidc_logout" - - # Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s - # https://github.com/mozilla/mozilla-django-oidc/issues/118 - # Using `/` because named urls don't work for this package - # https://github.com/mozilla/mozilla-django-oidc/issues/434 - LOGOUT_REDIRECT_URL = "route_manager:home" - - OIDC_OP_JWKS_ENDPOINT = os.environ.get( - "OIDC_OP_JWKS_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/certs", - ) - OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( - "OIDC_OP_AUTHORIZATION_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/auth", - ) - OIDC_OP_TOKEN_ENDPOINT = os.environ.get( - "OIDC_OP_TOKEN_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/token", - ) - OIDC_OP_USER_ENDPOINT = os.environ.get( - "OIDC_OP_USER_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", - ) - OIDC_RP_SIGN_ALGO = "RS256" - - OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") - OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") - -elif AUTH_METHOD == "local": - # https://docs.djangoproject.com/en/dev/ref/settings/#login-url - LOGIN_URL = "local_auth:login" - - # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "route_manager:home" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url - LOGOUT_URL = "local_auth:logout" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url - LOGOUT_REDIRECT_URL = "route_manager:home" -else: - raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") diff --git a/config/settings/test.py b/config/settings/test.py index 01f0cc8d..0163e07c 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -2,10 +2,8 @@ With these settings, tests run faster. """ -import os - from .base import * # noqa -from .base import AUTH_METHOD, env +from .base import env # GENERAL # ------------------------------------------------------------------------------ @@ -41,68 +39,3 @@ # Your stuff... # ------------------------------------------------------------------------------ -OIDC_OP_JWKS_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/certs" -OIDC_OP_AUTHORIZATION_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/auth" -OIDC_OP_TOKEN_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/token" -OIDC_OP_USER_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/userinfo" -OIDC_RP_SIGN_ALGO = "RS256" -OIDC_RP_CLIENT_ID = "" -OIDC_RP_CLIENT_SECRET = "" - -if AUTH_METHOD == "oidc": - # Extend middleware to add OIDC middleware - MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 - - # Extend middleware to add OIDC auth backend - AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 - - # https://docs.djangoproject.com/en/dev/ref/settings/#login-url - LOGIN_URL = "oidc_authentication_init" - - # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "route_manager:home" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url - LOGOUT_URL = "oidc_logout" - - # Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s - # https://github.com/mozilla/mozilla-django-oidc/issues/118 - # Using `/` because named urls don't work for this package - # https://github.com/mozilla/mozilla-django-oidc/issues/434 - LOGOUT_REDIRECT_URL = "route_manager:home" - - OIDC_OP_JWKS_ENDPOINT = os.environ.get( - "OIDC_OP_JWKS_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/certs", - ) - OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( - "OIDC_OP_AUTHORIZATION_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/auth", - ) - OIDC_OP_TOKEN_ENDPOINT = os.environ.get( - "OIDC_OP_TOKEN_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/token", - ) - OIDC_OP_USER_ENDPOINT = os.environ.get( - "OIDC_OP_USER_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", - ) - OIDC_RP_SIGN_ALGO = "RS256" - - OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") - OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") - -elif AUTH_METHOD == "local": - # https://docs.djangoproject.com/en/dev/ref/settings/#login-url - LOGIN_URL = "local_auth:login" - - # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "route_manager:home" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url - LOGOUT_URL = "local_auth:logout" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url - LOGOUT_REDIRECT_URL = "route_manager:home" -else: - raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") From 46d11fdb2d7be7984baf31cd9e0d1089a145f115 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 23:06:40 -0600 Subject: [PATCH 060/156] fix(test.py): OidcTest requires that we have these set even to dummy values We need this as soon as we load ESnetAuthBackend --- config/settings/test.py | 7 +++++++ scram/route_manager/tests/test_autocreate_admin.py | 0 2 files changed, 7 insertions(+) create mode 100644 scram/route_manager/tests/test_autocreate_admin.py diff --git a/config/settings/test.py b/config/settings/test.py index 0163e07c..44352142 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -39,3 +39,10 @@ # Your stuff... # ------------------------------------------------------------------------------ +OIDC_OP_JWKS_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/certs" +OIDC_OP_AUTHORIZATION_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/auth" +OIDC_OP_TOKEN_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/token" +OIDC_OP_USER_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/userinfo" +OIDC_RP_SIGN_ALGO = "RS256" +OIDC_RP_CLIENT_ID = "" +OIDC_RP_CLIENT_SECRET = "" diff --git a/scram/route_manager/tests/test_autocreate_admin.py b/scram/route_manager/tests/test_autocreate_admin.py new file mode 100644 index 00000000..e69de29b From d387849ab4d1efa63a0d30ec21ab83aa6642ddce Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 23:54:51 -0600 Subject: [PATCH 061/156] fix(OidcTest): OIDC vars required by our OidcTest case --- config/settings/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config/settings/test.py b/config/settings/test.py index 44352142..a4951b2c 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -39,6 +39,7 @@ # Your stuff... # ------------------------------------------------------------------------------ +# These variables are required by the ESnetAuthBackend called in our OidcTest case OIDC_OP_JWKS_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/certs" OIDC_OP_AUTHORIZATION_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/auth" OIDC_OP_TOKEN_ENDPOINT = "https://example.com/auth/realms/example/protocol/openid-connect/token" From 712815ca4b164e6017f37efbd47801ec5a802d15 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sun, 17 Nov 2024 23:55:57 -0600 Subject: [PATCH 062/156] build(sugar): this package needed to be updated to avoid a bug we started seeing while running tests See https://github.com/Teemu/pytest-sugar/issues/241 for more info --- requirements/local.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/local.txt b/requirements/local.txt index a0fbe77b..eef7c4bf 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -8,7 +8,7 @@ watchgod==0.8.2 # https://github.com/samuelcolvin/watchgod # Testing # ------------------------------------------------------------------------------ django-stubs==1.11.0 # https://github.com/typeddjango/django-stubs -pytest-sugar==0.9.4 # https://github.com/Frozenball/pytest-sugar +pytest-sugar==0.9.6 # https://github.com/Frozenball/pytest-sugar behave-django==1.4.0 # https://github.com/behave/behave-django # Documentation From 8bd0c441dc7587997d486351d7c56779d18ffa70 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 18 Nov 2024 00:01:26 -0600 Subject: [PATCH 063/156] test(autocreate_admin): make sure to test our autocreation of the admin user on first startup --- .../tests/test_autocreate_admin.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/scram/route_manager/tests/test_autocreate_admin.py b/scram/route_manager/tests/test_autocreate_admin.py index e69de29b..b5e7751b 100644 --- a/scram/route_manager/tests/test_autocreate_admin.py +++ b/scram/route_manager/tests/test_autocreate_admin.py @@ -0,0 +1,51 @@ +"""This file contains tests for the auto-creation of an admin user.""" + +import pytest +from django.contrib.auth.models import User +from django.contrib.messages import get_messages +from django.test import Client +from django.urls import reverse + +from scram.users.models import User + + +@pytest.mark.django_db +def test_autocreate_admin(settings): + """Test that an admin user is auto-created when AUTOCREATE_ADMIN is True.""" + + settings.AUTOCREATE_ADMIN = True + client = Client() + response = client.get(reverse("route_manager:home")) + assert response.status_code == 200 + assert User.objects.count() == 1 + user = User.objects.get(username="admin") + assert user.is_superuser + assert user.email == "admin@example.com" + messages = list(get_messages(response.wsgi_request)) + assert len(messages) == 2 + assert messages[0].level == 25 # SUCCESS + assert messages[1].level == 20 # INFO + + +@pytest.mark.django_db +def test_autocreate_admin_disabled(settings): + """Test that an admin user is not auto-created when AUTOCREATE_ADMIN is False.""" + + settings.AUTOCREATE_ADMIN = False + client = Client() + response = client.get(reverse("route_manager:home")) + assert response.status_code == 200 + assert User.objects.count() == 0 + + +@pytest.mark.django_db +def test_autocreate_admin_existing_user(settings): + """Test that an admin user is not auto-created when an existing user is present.""" + + settings.AUTOCREATE_ADMIN = True + User.objects.create_user("testuser", "test@example.com", "password") + client = Client() + response = client.get(reverse("route_manager:home")) + assert response.status_code == 200 + assert User.objects.count() == 1 + assert not User.objects.filter(username="admin").exists() From 1b8a44bbaebe85756f64caf2dbd13e9ce77e75a7 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 18 Nov 2024 09:10:18 -0600 Subject: [PATCH 064/156] Don't generate docs for the Django migrations. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index d4d4bad7..cd5336a0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,7 +48,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**/*.migrations.rst"] # -- Options for HTML output ------------------------------------------------- From c8d059b2bc9ae5d238a85d4e92d83771837aa6b9 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 18 Nov 2024 10:13:15 -0600 Subject: [PATCH 065/156] We covered the cookiecutter docs in more detail elsewhere. --- docs/cookiecutter.md | 71 -------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 docs/cookiecutter.md diff --git a/docs/cookiecutter.md b/docs/cookiecutter.md deleted file mode 100644 index 17445af4..00000000 --- a/docs/cookiecutter.md +++ /dev/null @@ -1,71 +0,0 @@ -This documentation was initially provided in the README.rst from the cookiecutter used to generate the project - -Settings --------- - -Moved to settings_. - -.. _settings: http://cookiecutter-django.readthedocs.io/en/latest/settings.html - -Basic Commands --------------- - -Setting Up Your Users -^^^^^^^^^^^^^^^^^^^^^ - -* To create a **normal user account**, just go to Sign Up and fill out the form. Once you submit it, you'll see a "Verify Your E-mail Address" page. Go to your console to see a simulated email verification message. Copy the link into your browser. Now the user's email should be verified and ready to go. - -* To create an **superuser account**, use this command:: - - $ python manage.py createsuperuser - -For convenience, you can keep your normal user logged in on Chrome and your superuser logged in on Firefox (or similar), so that you can see how the site behaves for both kinds of users. - -Type checks -^^^^^^^^^^^ - -Running type checks with mypy: - -:: - - $ mypy scram - -Test coverage -^^^^^^^^^^^^^ - -To run the tests, check your test coverage, and generate an HTML coverage report:: - - $ coverage run -m pytest - $ coverage html - $ open htmlcov/index.html - -Running tests with py.test -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:: - - $ pytest - -Live reloading and Sass CSS compilation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Moved to `Live reloading and SASS compilation`_. - -.. _`Live reloading and SASS compilation`: http://cookiecutter-django.readthedocs.io/en/latest/live-reloading-and-sass-compilation.html - - - -Deployment ----------- - -The following details how to deploy this application. - - - -Docker -^^^^^^ - -See detailed `cookiecutter-django Docker documentation`_. - -.. _`cookiecutter-django Docker documentation`: http://cookiecutter-django.readthedocs.io/en/latest/deployment-with-docker.html - From d28086114ee49499e47360f17f917b44e9f3f81f Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 18 Nov 2024 10:18:49 -0600 Subject: [PATCH 066/156] style(flake8): update docstrings and spacing for flake8 --- scram/local_auth/__init__.py | 1 + scram/local_auth/urls.py | 2 ++ scram/route_manager/tests/test_authorization.py | 3 ++- scram/route_manager/tests/test_autocreate_admin.py | 3 --- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scram/local_auth/__init__.py b/scram/local_auth/__init__.py index e69de29b..113a7bfa 100644 --- a/scram/local_auth/__init__.py +++ b/scram/local_auth/__init__.py @@ -0,0 +1 @@ +"""Local_auth is the app that holds urls we want to use with local Django auth.""" diff --git a/scram/local_auth/urls.py b/scram/local_auth/urls.py index 0d930946..56c7b727 100644 --- a/scram/local_auth/urls.py +++ b/scram/local_auth/urls.py @@ -1,3 +1,5 @@ +"""Register URLs for local auth known to Django, and the View that will handle each.""" + from django.contrib.auth.views import LoginView, LogoutView from django.urls import path diff --git a/scram/route_manager/tests/test_authorization.py b/scram/route_manager/tests/test_authorization.py index e1b7f83f..17de2b6d 100644 --- a/scram/route_manager/tests/test_authorization.py +++ b/scram/route_manager/tests/test_authorization.py @@ -132,8 +132,9 @@ def test_unauthorized_after_group_removal(self): self.assertEqual(response.status_code, 302) - class ESnetAuthBackendTest(TestCase): + """Define tests using OIDC authentication with our ESnetAuthBackend.""" + def setUp(self): """Create a sample OIDC user.""" self.client = Client() diff --git a/scram/route_manager/tests/test_autocreate_admin.py b/scram/route_manager/tests/test_autocreate_admin.py index b5e7751b..b9d2e432 100644 --- a/scram/route_manager/tests/test_autocreate_admin.py +++ b/scram/route_manager/tests/test_autocreate_admin.py @@ -12,7 +12,6 @@ @pytest.mark.django_db def test_autocreate_admin(settings): """Test that an admin user is auto-created when AUTOCREATE_ADMIN is True.""" - settings.AUTOCREATE_ADMIN = True client = Client() response = client.get(reverse("route_manager:home")) @@ -30,7 +29,6 @@ def test_autocreate_admin(settings): @pytest.mark.django_db def test_autocreate_admin_disabled(settings): """Test that an admin user is not auto-created when AUTOCREATE_ADMIN is False.""" - settings.AUTOCREATE_ADMIN = False client = Client() response = client.get(reverse("route_manager:home")) @@ -41,7 +39,6 @@ def test_autocreate_admin_disabled(settings): @pytest.mark.django_db def test_autocreate_admin_existing_user(settings): """Test that an admin user is not auto-created when an existing user is present.""" - settings.AUTOCREATE_ADMIN = True User.objects.create_user("testuser", "test@example.com", "password") client = Client() From 85c2f1e3cb52a17de15956beadc8a1696cdec9b1 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 18 Nov 2024 12:41:57 -0600 Subject: [PATCH 067/156] Switch sphinx to mkdocs --- docs/Makefile | 21 ++++++--------------- README.rst => docs/README.md | 33 ++++++++++----------------------- docs/reference/django.md | 3 +++ docs/reference/index.md | 7 +++++++ docs/reference/translator.md | 3 +++ mkdocs.yml | 20 ++++++++++++++++++++ requirements/local.txt | 12 ++++++++++-- 7 files changed, 59 insertions(+), 40 deletions(-) rename README.rst => docs/README.md (81%) create mode 100644 docs/reference/django.md create mode 100644 docs/reference/index.md create mode 100644 docs/reference/translator.md create mode 100644 mkdocs.yml diff --git a/docs/Makefile b/docs/Makefile index f7bac529..efd77582 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,29 +1,20 @@ -# Minimal makefile for Sphinx documentation -# - # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -c . +DOCBUILD ?= mkdocs SOURCEDIR = . -BUILDDIR = ./_build APP = /app -.PHONY: help livehtml apidocs Makefile +.PHONY: help livehtml Makefile # Put it first so that "make" without argument is like "make help". help: - @$(SPHINXBUILD) help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(DOCBUILD) -h "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) # Build, watch and serve docs with live reload livehtml: - sphinx-autobuild -b html --host 0.0.0.0 --port 7000 --watch $(APP) -c . $(SOURCEDIR) $(BUILDDIR)/html - -# Outputs rst files from django application code -apidocs: - sphinx-apidoc -o $(SOURCEDIR)/api /app + @$(DOCBUILD) serve -w $(SOURCEDIR) -# Catch-all target: route all unknown targets to Sphinx using the new +# Catch-all target: route all unknown targets to build # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -b $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @$(DOCBUILD) build diff --git a/README.rst b/docs/README.md similarity index 81% rename from README.rst rename to docs/README.md index 0b5ed230..e7e10ed7 100644 --- a/README.rst +++ b/docs/README.md @@ -1,30 +1,19 @@ -SCRAM -===== +# SCRAM Security Catch and Release Automation Manager -.. image:: https://coveralls.io/repos/github/esnet-security/SCRAM/badge.svg - :target: https://coveralls.io/github/esnet-security/SCRAM - :alt: Coveralls Code Coverage Stats -.. image:: https://img.shields.io/badge/built%20with-Cookiecutter%20Django-ff69b4.svg?logo=cookiecutter - :target: https://github.com/pydanny/cookiecutter-django/ - :alt: Built with Cookiecutter Django -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/ambv/black - :alt: Black code style +[]() +[]() +[]() -:License: BSD - -==== - -Overview -==== +## Overview SCRAM is a web based service to assist in automation of security data. There is a web interface as well as a REST API available. + The idea is to create actiontypes which allow you to take actions on the IPs/cidr networks you provide. -Components -==== +## Components + SCRAM utilizes ``docker compose`` to run the following stack in production: - nginx (as a webserver and static asset server) @@ -37,8 +26,7 @@ SCRAM utilizes ``docker compose`` to run the following stack in production: A predefined actiontype of "block" exists which utilizes bgp nullrouting to effectivley block any traffic you want to apply. You can add any other actiontypes via the admin page of the web interface dynamically, but keep in mind translator support would need to be added as well. -Installation -==== +## Installation To get a basic implementation up and running locally: @@ -58,8 +46,7 @@ To get a basic implementation up and running locally: - ``make django-open`` -*** Copyright Notice *** -==== +## Copyright Notice Security Catch and Release Automation Manager (SCRAM) Copyright (c) 2022, The Regents of the University of California, through Lawrence Berkeley diff --git a/docs/reference/django.md b/docs/reference/django.md new file mode 100644 index 00000000..be734554 --- /dev/null +++ b/docs/reference/django.md @@ -0,0 +1,3 @@ +# Route Manager + +::: scram.route_manager diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..fc771185 --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,7 @@ +# Module Reference + +The SCRAM ecosystem consists of two parts: + +A Django app, [route_manager](/reference/django) + +A translator service, [translator](/reference/translator) diff --git a/docs/reference/translator.md b/docs/reference/translator.md new file mode 100644 index 00000000..897d84e2 --- /dev/null +++ b/docs/reference/translator.md @@ -0,0 +1,3 @@ +# Translator + +::: translator diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..6ac26415 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,20 @@ +site_name: Security Catch and Release Automation Manager + +theme: + name: "material" + +plugins: + - search + - mkdocstrings: + handlers: + python: + options: + show_submodules: true + - section-index + +markdown_extensions: +# - pymdownx.highlight +# - deduplicate-toc +# - toc: +# permalink: "#" + diff --git a/requirements/local.txt b/requirements/local.txt index c2e5317c..72e40331 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -13,8 +13,16 @@ behave-django==1.4.0 # https://github.com/behave/behave-django # Documentation # ------------------------------------------------------------------------------ -sphinx==5.0.1 # https://github.com/sphinx-doc/sphinx -sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild +mkdocs==1.6.1 +mkdocs-autorefs==1.2.0 +mkdocs-gen-files==0.5.0 +mkdocs-get-deps==0.2.0 +mkdocs-literate-nav==0.6.1 +mkdocs-material==9.5.44 +mkdocs-material-extensions==1.3.1 +mkdocs-section-index==0.3.9 +mkdocstrings==0.27.0 +mkdocstrings-python==1.12.2 # Code quality # ------------------------------------------------------------------------------ From 25ed6737fb51a60cc4b53111d86d8fa66e96465e Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 18 Nov 2024 12:44:46 -0600 Subject: [PATCH 068/156] Bump Python for docs build --- compose/local/docs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/local/docs/Dockerfile b/compose/local/docs/Dockerfile index 9ab25f44..758ca263 100644 --- a/compose/local/docs/Dockerfile +++ b/compose/local/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim-buster +FROM python:3.12-slim-bookworm ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 From 35864599b0babc66ae44ec1313045b6ca4a847a6 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 18 Nov 2024 15:43:28 -0600 Subject: [PATCH 069/156] Link to Django docs --- .github/workflows/docs.yml | 2 +- mkdocs.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d3fbb051..223f58d5 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,7 +45,7 @@ jobs: uses: actions/upload-pages-artifact@v3 with: # Upload entire repository - path: 'docs/_build/html' + path: 'site/' - name: Deploy to GitHub Pages if: github.ref == 'refs/heads/main' diff --git a/mkdocs.yml b/mkdocs.yml index 6ac26415..82892980 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,12 +4,15 @@ theme: name: "material" plugins: - - search - mkdocstrings: handlers: python: + import: + - url: https://docs.djangoproject.com/en/4.2/_objects/ + base_url: https://docs.djangoproject.com/en/4.2/ options: show_submodules: true + - search - section-index markdown_extensions: From 2a4f8e5abfa38431d4993817787e1a7344b526ed Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 19 Nov 2024 12:01:54 -0600 Subject: [PATCH 070/156] refactor(login_redirect_url): no need to set this twice since both paths use the same value --- config/settings/base.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 20e9e558..ce4407c0 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -280,6 +280,9 @@ # Are you using local passwords or oidc? AUTH_METHOD = os.environ.get("SCRAM_AUTH_METHOD", "local").lower() +# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url +LOGIN_REDIRECT_URL = "route_manager:home" + logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware @@ -291,9 +294,6 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "oidc_authentication_init" - # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "route_manager:home" - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url LOGOUT_URL = "oidc_logout" @@ -328,9 +328,6 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "local_auth:login" - # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url - LOGIN_REDIRECT_URL = "route_manager:home" - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url LOGOUT_URL = "local_auth:logout" @@ -366,8 +363,6 @@ # This is the set of all the groups SCRAM_GROUPS = SCRAM_ADMIN_GROUPS + SCRAM_READWRITE_GROUPS + SCRAM_READONLY_GROUPS + SCRAM_DENIED_GROUPS -# This is the full set of groups - # How many entries to show PER Actiontype on the home page RECENT_LIMIT = 10 # What is the largest cidr range we'll accept entries for From 7fdb6abdb2a25551bc163083e95b470d9e0f632e Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Tue, 19 Nov 2024 22:23:24 -0600 Subject: [PATCH 071/156] refactor(OIDC-settings): move OIDC settings outside of the if statement this way we have defaults set and it doesn't harm anything to have them set but unused if we dont need them --- config/settings/base.py | 51 +++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index ce4407c0..766d785e 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -283,6 +283,30 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url LOGIN_REDIRECT_URL = "route_manager:home" +# Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s +# https://github.com/mozilla/mozilla-django-oidc/issues/118 +# Using `/` because named urls don't work for this package +# https://github.com/mozilla/mozilla-django-oidc/issues/434 +LOGOUT_REDIRECT_URL = "route_manager:home" + +OIDC_OP_JWKS_ENDPOINT = os.environ.get( + "OIDC_OP_JWKS_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/certs", +) +OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( + "OIDC_OP_AUTHORIZATION_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/auth", +) +OIDC_OP_TOKEN_ENDPOINT = os.environ.get( + "OIDC_OP_TOKEN_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/token", +) +OIDC_OP_USER_ENDPOINT = os.environ.get( + "OIDC_OP_USER_ENDPOINT", + "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", +) +OIDC_RP_SIGN_ALGO = "RS256" + logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware @@ -297,30 +321,6 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url LOGOUT_URL = "oidc_logout" - # Need to point somewhere otherwise /oidc/logout/ redirects to /oidc/logout/None which 404s - # https://github.com/mozilla/mozilla-django-oidc/issues/118 - # Using `/` because named urls don't work for this package - # https://github.com/mozilla/mozilla-django-oidc/issues/434 - LOGOUT_REDIRECT_URL = "route_manager:home" - - OIDC_OP_JWKS_ENDPOINT = os.environ.get( - "OIDC_OP_JWKS_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/certs", - ) - OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get( - "OIDC_OP_AUTHORIZATION_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/auth", - ) - OIDC_OP_TOKEN_ENDPOINT = os.environ.get( - "OIDC_OP_TOKEN_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/token", - ) - OIDC_OP_USER_ENDPOINT = os.environ.get( - "OIDC_OP_USER_ENDPOINT", - "https://example.com/auth/realms/example/protocol/openid-connect/userinfo", - ) - OIDC_RP_SIGN_ALGO = "RS256" - OIDC_RP_CLIENT_ID = os.environ.get("OIDC_RP_CLIENT_ID") OIDC_RP_CLIENT_SECRET = os.environ.get("OIDC_RP_CLIENT_SECRET") @@ -330,9 +330,6 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url LOGOUT_URL = "local_auth:logout" - - # https://docs.djangoproject.com/en/dev/ref/settings/#logout-redirect-url - LOGOUT_REDIRECT_URL = "route_manager:home" else: raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") From d1f44b55c2c9534d49f1c148f5cbc8dec6d222b3 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 08:34:34 -0600 Subject: [PATCH 072/156] build(drf_spectacular): we probably only need swagger available locally --- requirements/base.txt | 2 -- requirements/local.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 7577c200..c4aef7fc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -22,8 +22,6 @@ django-simple-history~=3.1.1 # Django REST Framework djangorestframework~=3.15 # https://github.com/encode/django-rest-framework django-cors-headers==3.13.0 # https://github.com/adamchainz/django-cors-headers -# DRF-spectacular for api documentation -drf-spectacular # https://github.com/tfranzel/drf-spectacular # OIDC # ------------------------------------------------------------------------------ diff --git a/requirements/local.txt b/requirements/local.txt index 2a6c95ae..63655dc3 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -15,6 +15,7 @@ behave-django==1.4.0 # https://github.com/behave/behave-django # ------------------------------------------------------------------------------ sphinx==5.0.1 # https://github.com/sphinx-doc/sphinx sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild +drf-spectacular # https://github.com/tfranzel/drf-spectacular # Code quality # ------------------------------------------------------------------------------ From c726e49b945b07d60d1287e2cea6b3096bd8166a Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 08:35:35 -0600 Subject: [PATCH 073/156] feat(swagger): set up swagger config bits --- config/settings/local.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/settings/local.py b/config/settings/local.py index 83b31dd8..8cc69a9d 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -36,7 +36,7 @@ # django-coverage-plugin # ------------------------------------------------------------------------------ # https://github.com/nedbat/django_coverage_plugin?tab=readme-ov-file#django-template-coveragepy-plugin -TEMPLATES[0]["OPTIONS"]['debug'] = True # noqa F405 +TEMPLATES[0]["OPTIONS"]["debug"] = True # noqa F405 # django-debug-toolbar # ------------------------------------------------------------------------------ @@ -67,11 +67,15 @@ REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAdminUser",), + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } # Behave Django testing framework INSTALLED_APPS += ["behave_django"] # noqa F405 +# Swagger related tooling +INSTALLED_APPS += ["drf_spectacular"] # noqa F405 + # AUTHENTICATION # ------------------------------------------------------------------------------ # We shouldn't be using OIDC in local dev mode as of now, but might be worth pursuing later From d3e4a6aec30f4c4afe7ce83d411b6be477ff3946 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Thu, 21 Nov 2024 08:36:07 -0600 Subject: [PATCH 074/156] Remove commented out bits. --- docs/conf.py | 63 ---------------------------------------------------- mkdocs.yml | 7 ------ 2 files changed, 70 deletions(-) delete mode 100644 docs/conf.py diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index cd5336a0..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,63 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - -import os -import sys - -import django - -if os.getenv("READTHEDOCS", default=False) == "True": - sys.path.insert(0, os.path.abspath("..")) - os.environ["DJANGO_READ_DOT_ENV_FILE"] = "True" - os.environ["USE_DOCKER"] = "no" -else: - sys.path.insert(0, os.path.abspath("/app")) -os.environ["DATABASE_URL"] = "sqlite:///readthedocs.db" -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") -django.setup() - -# -- Project information ----------------------------------------------------- - -project = "SCRAM" -copyright = """2021, Sam Oehlert""" -author = "Sam Oehlert" - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", -] - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**/*.migrations.rst"] - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "alabaster" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ["_static"] diff --git a/mkdocs.yml b/mkdocs.yml index 82892980..704db80c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,10 +14,3 @@ plugins: show_submodules: true - search - section-index - -markdown_extensions: -# - pymdownx.highlight -# - deduplicate-toc -# - toc: -# permalink: "#" - From 8e29b32b7428c0ec0018406bbd1d0220e57c26b5 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 08:36:39 -0600 Subject: [PATCH 075/156] feat(swagger): set up our views to use swagger and expose the endpoints if in local mode --- config/urls.py | 8 ++++++++ scram/route_manager/api/views.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/config/urls.py b/config/urls.py index ec9e5b7b..709eefe0 100644 --- a/config/urls.py +++ b/config/urls.py @@ -4,6 +4,7 @@ from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.urls import include, path from django.views import defaults as default_views +from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView from rest_framework.authtoken.views import obtain_auth_token from .api_router import app_name @@ -69,3 +70,10 @@ import debug_toolbar urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns + +if "drf_spectacular" in settings.INSTALLED_APPS: + urlpatterns += [ + path("schema/", SpectacularAPIView.as_view(), name="schema"), + path("schema/swagger-ui/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), + path("schema/redoc/", SpectacularRedocView.as_view(url_name="schema"), name="redoc"), + ] diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index ad26127e..32a9e07e 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -10,6 +10,7 @@ from django.db.models import Q from django.http import Http404 from django.utils.dateparse import parse_datetime +from drf_spectacular.utils import extend_schema from rest_framework import status, viewsets from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response @@ -21,6 +22,10 @@ channel_layer = get_channel_layer() +@extend_schema( + description="API endpoint for actiontypes", + responses={200: "Success"}, +) class ActionTypeViewSet(viewsets.ReadOnlyModelViewSet): """Lookup ActionTypes by name when authenticated, and bind to the serializer.""" @@ -30,6 +35,10 @@ class ActionTypeViewSet(viewsets.ReadOnlyModelViewSet): lookup_field = "name" +@extend_schema( + description="API endpoint for ignore entries", + responses={200: "Success"}, +) class IgnoreEntryViewSet(viewsets.ModelViewSet): """Lookup IgnoreEntries by route when authenticated, and bind to the serializer.""" @@ -39,6 +48,10 @@ class IgnoreEntryViewSet(viewsets.ModelViewSet): lookup_field = "route" +@extend_schema( + description="API endpoint for clients", + responses={200: "Success"}, +) class ClientViewSet(viewsets.ModelViewSet): """Lookup Client by hostname on POSTs regardless of authentication, and bind to the serializer.""" @@ -50,6 +63,10 @@ class ClientViewSet(viewsets.ModelViewSet): http_method_names = ["post"] +@extend_schema( + description="API endpoint for entries", + responses={200: "Success"}, +) class EntryViewSet(viewsets.ModelViewSet): """Lookup Entry when authenticated, and bind to the serializer.""" From d3b66f83b7dd210e17194b25e3dd71034c464649 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 17:53:52 -0600 Subject: [PATCH 076/156] refactor(URLs): make the swagger URLs available all the time --- config/urls.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/urls.py b/config/urls.py index 709eefe0..0a4648b7 100644 --- a/config/urls.py +++ b/config/urls.py @@ -45,6 +45,13 @@ path("auth-token/", obtain_auth_token), ] +# Swagger OpenAPI URLs +urlpatterns += [ + path("schema/", SpectacularAPIView.as_view(), name="schema"), + path("schema/swagger-ui/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), + path("schema/redoc/", SpectacularRedocView.as_view(url_name="schema"), name="redoc"), +] + if settings.DEBUG: # This allows the error pages to be debugged during development, just visit # these url in browser to see how these error pages look like. @@ -70,10 +77,3 @@ import debug_toolbar urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns - -if "drf_spectacular" in settings.INSTALLED_APPS: - urlpatterns += [ - path("schema/", SpectacularAPIView.as_view(), name="schema"), - path("schema/swagger-ui/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), - path("schema/redoc/", SpectacularRedocView.as_view(url_name="schema"), name="redoc"), - ] From ec115fc936e5a355485c4a56d8c59b921e463c6c Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:11:02 -0600 Subject: [PATCH 077/156] refactor(swagger): move swagger settings to base.py so its available in prod mode too i was incorrectly overriding the REST_FRAMEWORK dict and that's why it kept failing every time i moved settings to base. now i am only overriding one key in here and things work again --- config/settings/base.py | 3 +++ config/settings/local.py | 11 ++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 766d785e..edaa90f9 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -264,6 +264,8 @@ # django-rest-framework # ------------------------------------------------------------------------------- # django-rest-framework - https://www.django-rest-framework.org/api-guide/settings/ +# Swagger related tooling +INSTALLED_APPS += ["drf_spectacular"] # noqa F405 REST_FRAMEWORK = { "DEFAULT_AUTHENTICATION_CLASSES": ( "rest_framework.authentication.SessionAuthentication", @@ -271,6 +273,7 @@ ), "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), "TEST_REQUEST_DEFAULT_FORMAT": "json", + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } # django-cors-headers - https://github.com/adamchainz/django-cors-headers#setup diff --git a/config/settings/local.py b/config/settings/local.py index 8cc69a9d..8d81f04d 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -1,5 +1,5 @@ from .base import * # noqa -from .base import AUTH_METHOD, env +from .base import AUTH_METHOD, REST_FRAMEWORK, env # GENERAL # ------------------------------------------------------------------------------ @@ -64,18 +64,11 @@ # Your stuff... # ------------------------------------------------------------------------------ - -REST_FRAMEWORK = { - "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAdminUser",), - "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", -} +REST_FRAMEWORK["DEFAULT_PERMISSION_CLASSES"] = ("rest_framework.permissions.IsAdminUser",) # Behave Django testing framework INSTALLED_APPS += ["behave_django"] # noqa F405 -# Swagger related tooling -INSTALLED_APPS += ["drf_spectacular"] # noqa F405 - # AUTHENTICATION # ------------------------------------------------------------------------------ # We shouldn't be using OIDC in local dev mode as of now, but might be worth pursuing later From f14521f0dcd498614199da581648b9022835c365 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:11:44 -0600 Subject: [PATCH 078/156] feat(serializer): use appropriate serializers in success response instead of just a success string --- scram/route_manager/api/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 32a9e07e..46e12021 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -24,7 +24,7 @@ @extend_schema( description="API endpoint for actiontypes", - responses={200: "Success"}, + responses={200: ActionTypeSerializer}, ) class ActionTypeViewSet(viewsets.ReadOnlyModelViewSet): """Lookup ActionTypes by name when authenticated, and bind to the serializer.""" @@ -37,7 +37,7 @@ class ActionTypeViewSet(viewsets.ReadOnlyModelViewSet): @extend_schema( description="API endpoint for ignore entries", - responses={200: "Success"}, + responses={200: IgnoreEntrySerializer}, ) class IgnoreEntryViewSet(viewsets.ModelViewSet): """Lookup IgnoreEntries by route when authenticated, and bind to the serializer.""" @@ -50,7 +50,7 @@ class IgnoreEntryViewSet(viewsets.ModelViewSet): @extend_schema( description="API endpoint for clients", - responses={200: "Success"}, + responses={200: ClientSerializer}, ) class ClientViewSet(viewsets.ModelViewSet): """Lookup Client by hostname on POSTs regardless of authentication, and bind to the serializer.""" @@ -65,7 +65,7 @@ class ClientViewSet(viewsets.ModelViewSet): @extend_schema( description="API endpoint for entries", - responses={200: "Success"}, + responses={200: EntrySerializer}, ) class EntryViewSet(viewsets.ModelViewSet): """Lookup Entry when authenticated, and bind to the serializer.""" From 3399ea0e84d053922a7fa6c3f085964e9836da14 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:13:38 -0600 Subject: [PATCH 079/156] feat(CustomCidrAddressField): create a custom field that inherits from rest_framework.CidrAddressField so that we can tell swagger how to deal with this type of field --- scram/route_manager/api/serializers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index 6baf7d95..d097c67b 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -2,6 +2,7 @@ import logging +from drf_spectacular.utils import extend_schema_field from netfields import rest_framework from rest_framework import serializers from rest_framework.fields import CurrentUserDefault @@ -12,6 +13,11 @@ logger = logging.getLogger(__name__) +@extend_schema_field(field={"type": "string", "format": "cidr"}) +class CustomCidrAddressField(rest_framework.CidrAddressField): + pass + + class ActionTypeSerializer(serializers.ModelSerializer): """This serializer defines no new fields.""" @@ -25,7 +31,7 @@ class Meta: class RouteSerializer(serializers.ModelSerializer): """Exposes route as a CIDR field.""" - route = rest_framework.CidrAddressField() + route = CustomCidrAddressField() class Meta: """Maps to the Route model, and specifies the fields exposed by the API.""" @@ -52,7 +58,7 @@ class EntrySerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( view_name="api:v1:entry-detail", lookup_url_kwarg="pk", lookup_field="route" ) - route = rest_framework.CidrAddressField() + route = CustomCidrAddressField() actiontype = serializers.CharField(default="block") if CurrentUserDefault(): # This is set if we are calling this serializer from WUI @@ -90,6 +96,8 @@ def create(self, validated_data): class IgnoreEntrySerializer(serializers.ModelSerializer): """This serializer defines no new fields.""" + route = CustomCidrAddressField() + class Meta: """Maps to the IgnoreEntry model, and specifies the fields exposed by the API.""" From 939e0098aa70fa655d0d3202030cac330f5e36b8 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:28:22 -0600 Subject: [PATCH 080/156] style(flake8): docstring add missing docstring to our new custom field class --- scram/route_manager/api/serializers.py | 2 ++ scram/route_manager/tests/test_swagger.py | 0 2 files changed, 2 insertions(+) create mode 100644 scram/route_manager/tests/test_swagger.py diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index d097c67b..c42f0067 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -15,6 +15,8 @@ @extend_schema_field(field={"type": "string", "format": "cidr"}) class CustomCidrAddressField(rest_framework.CidrAddressField): + """This serializer defines a wrapper field so swagger can properly handle the inherited field""" + pass diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py new file mode 100644 index 00000000..e69de29b From 55ff3ec0adfea13ecfeced7f69d60ee1edb985f3 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:31:59 -0600 Subject: [PATCH 081/156] style(flake8): flake8 wants docstrings --- scram/route_manager/api/serializers.py | 2 +- scram/route_manager/tests/test_swagger.py | 47 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index c42f0067..ba3cae4b 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -15,7 +15,7 @@ @extend_schema_field(field={"type": "string", "format": "cidr"}) class CustomCidrAddressField(rest_framework.CidrAddressField): - """This serializer defines a wrapper field so swagger can properly handle the inherited field""" + """This serializer defines a wrapper field so swagger can properly handle the inherited field.""" pass diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py index e69de29b..6ffbf00b 100644 --- a/scram/route_manager/tests/test_swagger.py +++ b/scram/route_manager/tests/test_swagger.py @@ -0,0 +1,47 @@ +import pytest +from django.urls import reverse + + +@pytest.mark.django_db +def test_swagger_api(client): + """Test that the Swagger API endpoint returns a successful response.""" + + url = reverse("swagger-ui") + response = client.get(url) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_swagger_api_entry_list(client): + """Test that the Swagger API endpoint for the entry list returns a successful response.""" + + url = reverse("swagger-ui") + "?url=/api/v1/entry-list" + response = client.get(url) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_swagger_api_entry_detail(client): + """Test that the Swagger API endpoint for an entry detail returns a successful response.""" + + url = reverse("swagger-ui") + "?url=/api/v1/entry-detail/1" + response = client.get(url) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_swagger_api_ignore_entry_list(client): + """Test that the Swagger API endpoint for the ignore entry list returns a successful response.""" + + url = reverse("swagger-ui") + "?url=/api/v1/ignoreentry-list" + response = client.get(url) + assert response.status_code == 200 + + +@pytest.mark.django_db +def test_swagger_api_ignore_entry_detail(client): + """Test that the Swagger API endpoint for an ignore entry detail returns a successful response.""" + + url = reverse("swagger-ui") + "?url=/api/v1/ignoreentry-detail/1" + response = client.get(url) + assert response.status_code == 200 From 79bc45545f0d1113150b6f61bfaa4aa5e7e20bf1 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:36:41 -0600 Subject: [PATCH 082/156] test(swagger): test our swagger endpoints to make sure we can access them --- scram/route_manager/tests/test_swagger.py | 35 ++++++----------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py index 6ffbf00b..af820f21 100644 --- a/scram/route_manager/tests/test_swagger.py +++ b/scram/route_manager/tests/test_swagger.py @@ -1,3 +1,5 @@ +"""This file contains tests for the swagger API endpoints.""" + import pytest from django.urls import reverse @@ -5,43 +7,22 @@ @pytest.mark.django_db def test_swagger_api(client): """Test that the Swagger API endpoint returns a successful response.""" - url = reverse("swagger-ui") response = client.get(url) assert response.status_code == 200 @pytest.mark.django_db -def test_swagger_api_entry_list(client): - """Test that the Swagger API endpoint for the entry list returns a successful response.""" - - url = reverse("swagger-ui") + "?url=/api/v1/entry-list" - response = client.get(url) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_swagger_api_entry_detail(client): - """Test that the Swagger API endpoint for an entry detail returns a successful response.""" - - url = reverse("swagger-ui") + "?url=/api/v1/entry-detail/1" - response = client.get(url) - assert response.status_code == 200 - - -@pytest.mark.django_db -def test_swagger_api_ignore_entry_list(client): - """Test that the Swagger API endpoint for the ignore entry list returns a successful response.""" - - url = reverse("swagger-ui") + "?url=/api/v1/ignoreentry-list" +def test_redoc_api(client): + """Test that the Redoc API endpoint returns a successful response.""" + url = reverse("redoc") response = client.get(url) assert response.status_code == 200 @pytest.mark.django_db -def test_swagger_api_ignore_entry_detail(client): - """Test that the Swagger API endpoint for an ignore entry detail returns a successful response.""" - - url = reverse("swagger-ui") + "?url=/api/v1/ignoreentry-detail/1" +def test_schema_api(client): + """Test that the Schema API endpoint returns a successful response.""" + url = reverse("schema") response = client.get(url) assert response.status_code == 200 From 4400790734b4baa2768f18e30d7ac961f1b7ed6c Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Thu, 21 Nov 2024 22:47:46 -0600 Subject: [PATCH 083/156] refactor(swagger): put swagger package in base instead of local requirements --- requirements/base.txt | 1 + requirements/local.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index c4aef7fc..b43b582a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -22,6 +22,7 @@ django-simple-history~=3.1.1 # Django REST Framework djangorestframework~=3.15 # https://github.com/encode/django-rest-framework django-cors-headers==3.13.0 # https://github.com/adamchainz/django-cors-headers +drf-spectacular # https://github.com/tfranzel/drf-spectacular # OIDC # ------------------------------------------------------------------------------ diff --git a/requirements/local.txt b/requirements/local.txt index 63655dc3..2a6c95ae 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -15,7 +15,6 @@ behave-django==1.4.0 # https://github.com/behave/behave-django # ------------------------------------------------------------------------------ sphinx==5.0.1 # https://github.com/sphinx-doc/sphinx sphinx-autobuild==2021.3.14 # https://github.com/GaretJax/sphinx-autobuild -drf-spectacular # https://github.com/tfranzel/drf-spectacular # Code quality # ------------------------------------------------------------------------------ From 2816faf0c3a0f6cda810c88e8e1e96de2b816885 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 22 Nov 2024 14:17:36 -0600 Subject: [PATCH 084/156] test(swagger): make sure swagger is not only up, but serving out our API docs --- scram/route_manager/tests/test_swagger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py index af820f21..cd380ef2 100644 --- a/scram/route_manager/tests/test_swagger.py +++ b/scram/route_manager/tests/test_swagger.py @@ -26,3 +26,5 @@ def test_schema_api(client): url = reverse("schema") response = client.get(url) assert response.status_code == 200 + expected_strings = [b"entries", b"actiontypes", b"ignore_entries", b"user"] + assert all(string in response.content for string in expected_strings) From 26dd224890330695bb088c054024b0c9a4cfd3e3 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Fri, 22 Nov 2024 14:28:19 -0600 Subject: [PATCH 085/156] test(bytestrings): be more explicit about our bytestrings to make sure we are testing for both "entries" and "ignore_entries" separately --- scram/route_manager/tests/test_swagger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py index cd380ef2..fc1c0fe5 100644 --- a/scram/route_manager/tests/test_swagger.py +++ b/scram/route_manager/tests/test_swagger.py @@ -26,5 +26,5 @@ def test_schema_api(client): url = reverse("schema") response = client.get(url) assert response.status_code == 200 - expected_strings = [b"entries", b"actiontypes", b"ignore_entries", b"user"] + expected_strings = [b"/entries/", b"/actiontypes/", b"/ignore_entries/", b"/users/"] assert all(string in response.content for string in expected_strings) From 06b42a541bd3e71984640da04a17ff12e661f6f2 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:24:10 -0600 Subject: [PATCH 086/156] First pass at ruff and fixes. --- pyproject.toml | 16 ++++++-- scram/route_manager/api/exceptions.py | 2 +- .../tests/acceptance/steps/common.py | 38 +++++++++---------- .../tests/acceptance/steps/ip.py | 8 ++-- .../tests/acceptance/steps/translator.py | 4 +- .../tests/test_autocreate_admin.py | 1 - scram/route_manager/tests/test_websockets.py | 2 +- 7 files changed, 40 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 910571d6..3edb2451 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,21 @@ exclude_also = [ "if __name__ == .__main__.:", ] -# ==== black ==== -[tool.black] +# ===== ruff ==== +[tool.ruff] +# Exclude a variety of commonly ignored directories. +exclude = [ + "migrations", +] + line-length = 119 -target-version = ['py311'] +target-version = 'py312' +[tool.ruff.lint] +select = [ + # pycodestyle + "E", +] # ==== isort ==== [tool.isort] diff --git a/scram/route_manager/api/exceptions.py b/scram/route_manager/api/exceptions.py index 74d6f2e2..0445dbe4 100644 --- a/scram/route_manager/api/exceptions.py +++ b/scram/route_manager/api/exceptions.py @@ -10,7 +10,7 @@ class PrefixTooLarge(APIException): v4_min_prefix = getattr(settings, "V4_MINPREFIX", 0) v6_min_prefix = getattr(settings, "V6_MINPREFIX", 0) status_code = 400 - default_detail = f"You've supplied too large of a network. settings.V4_MINPREFIX = {v4_min_prefix} settings.V6_MINPREFIX = {v6_min_prefix}" # noqa: 501 + default_detail = f"You've supplied too large of a network. settings.V4_MINPREFIX = {v4_min_prefix} settings.V6_MINPREFIX = {v6_min_prefix}" # noqa: E501 default_code = "prefix_too_large" diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index c7c03f8c..cc732b15 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -13,7 +13,7 @@ @given("a {name} actiontype is defined") -def step_impl(context, name): +def create_actiontype(context, name): """Create an actiontype of that name.""" context.channel_layer = get_channel_layer() async_to_sync(context.channel_layer.group_send)( @@ -28,7 +28,7 @@ def step_impl(context, name): @given("a client with {name} authorization") -def step_impl(context, name): +def create_authed_client(context, name): """Create a client and authorize it for that action type.""" at, created = ActionType.objects.get_or_create(name=name) authorized_client = Client.objects.create( @@ -40,7 +40,7 @@ def step_impl(context, name): @given("a client without {name} authorization") -def step_impl(context, name): +def create_unauthed_client(context, name): """Create a client that has no authorized action types.""" unauthorized_client = Client.objects.create( hostname="unauthorized_client.es.net", @@ -50,26 +50,26 @@ def step_impl(context, name): @when("we're logged in") -def step_impl(context): +def login(context): """Login.""" context.test.client.login(username="user", password="password") @when("the CIDR prefix limits are {v4_minprefix:d} and {v6_minprefix:d}") -def step_impl(context, v4_minprefix, v6_minprefix): +def set_cidr_limit(context, v4_minprefix, v6_minprefix): """Override our settings with the provided values.""" conf.settings.V4_MINPREFIX = v4_minprefix conf.settings.V6_MINPREFIX = v6_minprefix @then("we get a {status_code:d} status code") -def step_impl(context, status_code): +def check_status_code(context, status_code): """Ensure the status code response matches the expected value.""" context.test.assertEqual(context.response.status_code, status_code) @when("we add the entry {value:S}") -def step_impl(context, value): +def add_entry(context, value): """Block the provided route.""" context.response = context.test.client.post( reverse("api:v1:entry-list"), @@ -86,7 +86,7 @@ def step_impl(context, value): @when("we add the entry {value:S} with comment {comment}") -def step_impl(context, value, comment): +def add_entry_with_comment(context, value, comment): """Block the provided route and add a comment.""" context.response = context.test.client.post( reverse("api:v1:entry-list"), @@ -102,7 +102,7 @@ def step_impl(context, value, comment): @when("we add the entry {value:S} with expiration {exp:S}") -def step_impl(context, value, exp): +def add_entry_with_absolute_expiration(context, value, exp): """Block the provided route and add an absolute expiration datetime.""" context.response = context.test.client.post( reverse("api:v1:entry-list"), @@ -119,7 +119,7 @@ def step_impl(context, value, exp): @when("we add the entry {value:S} with expiration in {secs:d} seconds") -def step_impl(context, value, secs): +def add_entry_with_relative_expiration(context, value, secs): """Block the provided route and add a relative expiration.""" td = datetime.timedelta(seconds=secs) expiration = datetime.datetime.now() + td @@ -139,19 +139,19 @@ def step_impl(context, value, secs): @step("we wait {secs:d} seconds") -def step_impl(context, secs): +def wait(context, secs): """Wait to allow messages to propagate.""" time.sleep(secs) @then("we remove expired entries") -def step_impl(context): +def remove_expired(context): """Call the function that removes expired entries.""" context.response = context.test.client.get(reverse("route_manager:process-expired")) @when("we add the ignore entry {value:S}") -def step_impl(context, value): +def add_ignore_entry(context, value): """Add an IgnoreEntry with the specified route.""" context.response = context.test.client.post( reverse("api:v1:ignoreentry-list"), {"route": value, "comment": "test api"} @@ -159,19 +159,19 @@ def step_impl(context, value): @when("we remove the {model} {value}") -def step_impl(context, model, value): +def remove_an_object(context, model, value): """Remove any model object with the matching value.""" context.response = context.test.client.delete(reverse(f"api:v1:{model.lower()}-detail", args=[value])) @when("we list the {model}s") -def step_impl(context, model): +def list_objects(context, model): """List all objects of an arbitrary model.""" context.response = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) @when("we update the {model} {value_from} to {value_to}") -def step_impl(context, model, value_from, value_to): +def update_object(context, model, value_from, value_to): """Modify any model object with the matching value to the new value instead.""" context.response = context.test.client.patch( reverse(f"api:v1:{model.lower()}-detail", args=[value_from]), @@ -180,7 +180,7 @@ def step_impl(context, model, value_from, value_to): @then("the number of {model}s is {num:d}") -def step_impl(context, model, num): +def count_objects(context, model, num): """Count the number of objects of an arbitrary model.""" objs = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) context.test.assertEqual(len(objs.json()), num) @@ -190,7 +190,7 @@ def step_impl(context, model, num): @then("{value} is one of our list of {model}s") -def step_impl(context, value, model): +def check_object(context, value, model): """Ensure that the arbitrary model has an object with the specified value.""" objs = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) @@ -206,7 +206,7 @@ def step_impl(context, value, model): @when("we register a client named {hostname} with the uuid of {uuid}") -def step_impl(context, hostname, uuid): +def add_client(context, hostname, uuid): """Create a client with a specific UUID.""" context.response = context.test.client.post( reverse("api:v1:client-list"), diff --git a/scram/route_manager/tests/acceptance/steps/ip.py b/scram/route_manager/tests/acceptance/steps/ip.py index 8154f04b..34536a86 100644 --- a/scram/route_manager/tests/acceptance/steps/ip.py +++ b/scram/route_manager/tests/acceptance/steps/ip.py @@ -7,7 +7,7 @@ @then("{route} is contained in our list of {model}s") -def step_impl(context, route, model): +def check_route(context, route, model): """Perform a CIDR match on the matching object.""" objs = context.test.client.get(reverse(f"api:v1:{model.lower()}-list")) ip_target = ipaddress.ip_address(route) @@ -23,7 +23,7 @@ def step_impl(context, route, model): @when("we query for {ip}") -def step_impl(context, ip): +def check_ip(context, ip): """Find an Entry for the specified IP.""" try: context.response = context.test.client.get(reverse("api:v1:entry-detail", args=[ip])) @@ -34,13 +34,13 @@ def step_impl(context, ip): @then("we get a ValueError") -def step_impl(context): +def check_error(context): """Ensure we received a ValueError exception.""" assert isinstance(context.queryException, ValueError) @then("the change entry for {value:S} is {comment}") -def step_impl(context, value, comment): +def check_comment(context, value, comment): """Verify the comment for the Entry.""" try: objs = context.test.client.get(reverse("api:v1:entry-detail", args=[value])) diff --git a/scram/route_manager/tests/acceptance/steps/translator.py b/scram/route_manager/tests/acceptance/steps/translator.py index c6c2ff49..9fdecb3a 100644 --- a/scram/route_manager/tests/acceptance/steps/translator.py +++ b/scram/route_manager/tests/acceptance/steps/translator.py @@ -23,13 +23,13 @@ async def query_translator(route, actiontype, is_announced=True): @then("{route} is announced by {actiontype} translators") @async_run_until_complete -async def step_impl(context, route, actiontype): +async def check_blocked(context, route, actiontype): """Ensure the specified route is currently blocked.""" await query_translator(route, actiontype) @then("{route} is not announced by {actiontype} translators") @async_run_until_complete -async def step_impl(context, route, actiontype): +async def check_unblocked(context, route, actiontype): """Ensure the specified route is currently unblocked.""" await query_translator(route, actiontype, is_announced=False) diff --git a/scram/route_manager/tests/test_autocreate_admin.py b/scram/route_manager/tests/test_autocreate_admin.py index b9d2e432..a550e3cb 100644 --- a/scram/route_manager/tests/test_autocreate_admin.py +++ b/scram/route_manager/tests/test_autocreate_admin.py @@ -1,7 +1,6 @@ """This file contains tests for the auto-creation of an admin user.""" import pytest -from django.contrib.auth.models import User from django.contrib.messages import get_messages from django.test import Client from django.urls import reverse diff --git a/scram/route_manager/tests/test_websockets.py b/scram/route_manager/tests/test_websockets.py index 306e772b..d8b58bc4 100644 --- a/scram/route_manager/tests/test_websockets.py +++ b/scram/route_manager/tests/test_websockets.py @@ -151,7 +151,7 @@ async def test_add_v6(self): class TranslatorDontCrossTheStreamsTestCase(TestTranslatorBaseCase): - """Two translators in the same group, two in another group, single IP, ensure we get only the messages we expect.""" + """Two translators in one group, two in another group, single IP, ensure we get only the messages we expect.""" def local_setUp(self): """Define the actions and what we expect.""" From d42e1fc33453331123dd1444a2ff95a9f3638a47 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:26:51 -0600 Subject: [PATCH 087/156] Move workflow to ruff --- .github/workflows/ruff.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/ruff.yml diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 00000000..69d587c3 --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,34 @@ +name: Run ruff + +on: + push: + branches: + - '**' + pull_request: + branches: + - main + - develop + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + flake8: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache Docker images. + uses: ScribeMD/docker-cache@0.3.7 + with: + key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }} + + - name: Install dependencies + run: pip install ruff + + - name: Fail if we have any style errors + run: ruff check + + - name: Fail if the code is not formatted correctly (like Black) + run: ruff format --diff From 1901c24e87ed3849523bbcc5aa9e08d290ec29f8 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:27:07 -0600 Subject: [PATCH 088/156] Ruff format fixes --- config/asgi.py | 1 + config/wsgi.py | 1 + 2 files changed, 2 insertions(+) diff --git a/config/asgi.py b/config/asgi.py index 98a16ce2..c261d139 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -7,6 +7,7 @@ https://docs.djangoproject.com/en/dev/howto/deployment/asgi/ """ + import logging import os import sys diff --git a/config/wsgi.py b/config/wsgi.py index 98830366..37c0e92e 100644 --- a/config/wsgi.py +++ b/config/wsgi.py @@ -13,6 +13,7 @@ framework. """ + import os import sys from pathlib import Path From 7cb63c0bf441e2eee8819a1946fdc56bb1ca5ba3 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:34:55 -0600 Subject: [PATCH 089/156] Enable pyflakes and pyupgrade --- .github/workflows/flake8.yml | 33 ---------- .github/workflows/ruff.yml | 7 +- merge_production_dotenvs_in_dotenv.py | 65 ------------------- pyproject.toml | 5 +- .../route_manager/authentication_backends.py | 2 +- scram/users/tests/factories.py | 3 +- translator/gobgp.py | 2 +- 7 files changed, 8 insertions(+), 109 deletions(-) delete mode 100644 .github/workflows/flake8.yml delete mode 100644 merge_production_dotenvs_in_dotenv.py diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index 887ebc50..00000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Run flake8 - -on: - push: - branches: - - '**' - pull_request: - branches: - - main - - develop - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - flake8: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Cache Docker images. - uses: ScribeMD/docker-cache@0.3.7 - with: - key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }} - - - name: Install dependencies - run: | - pip install flake8 flake8-docstrings - - - name: Run flake8 - run: | - flake8 scram translator diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 69d587c3..2a58b8bd 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -13,17 +13,12 @@ on: workflow_dispatch: jobs: - flake8: + ruff: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - - name: Cache Docker images. - uses: ScribeMD/docker-cache@0.3.7 - with: - key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yaml') }} - - name: Install dependencies run: pip install ruff diff --git a/merge_production_dotenvs_in_dotenv.py b/merge_production_dotenvs_in_dotenv.py deleted file mode 100644 index b66558c3..00000000 --- a/merge_production_dotenvs_in_dotenv.py +++ /dev/null @@ -1,65 +0,0 @@ -import os -from pathlib import Path -from typing import Sequence - -import pytest - -ROOT_DIR_PATH = Path(__file__).parent.resolve() -PRODUCTION_DOTENVS_DIR_PATH = ROOT_DIR_PATH / ".envs" / ".production" -PRODUCTION_DOTENV_FILE_PATHS = [ - PRODUCTION_DOTENVS_DIR_PATH / ".django", - PRODUCTION_DOTENVS_DIR_PATH / ".postgres", -] -DOTENV_FILE_PATH = ROOT_DIR_PATH / ".env" - - -def merge(output_file_path: str, merged_file_paths: Sequence[str], append_linesep: bool = True) -> None: - with open(output_file_path, "w") as output_file: - for merged_file_path in merged_file_paths: - with open(merged_file_path, "r") as merged_file: - merged_file_content = merged_file.read() - output_file.write(merged_file_content) - if append_linesep: - output_file.write(os.linesep) - - -def main(): - merge(DOTENV_FILE_PATH, PRODUCTION_DOTENV_FILE_PATHS) - - -@pytest.mark.parametrize("merged_file_count", range(3)) -@pytest.mark.parametrize("append_linesep", [True, False]) -def test_merge(tmpdir_factory, merged_file_count: int, append_linesep: bool): - tmp_dir_path = Path(str(tmpdir_factory.getbasetemp())) - - output_file_path = tmp_dir_path / ".env" - - expected_output_file_content = "" - merged_file_paths = [] - for i in range(merged_file_count): - merged_file_ord = i + 1 - - merged_filename = ".service{}".format(merged_file_ord) - merged_file_path = tmp_dir_path / merged_filename - - merged_file_content = merged_filename * merged_file_ord - - with open(merged_file_path, "w+") as file: - file.write(merged_file_content) - - expected_output_file_content += merged_file_content - if append_linesep: - expected_output_file_content += os.linesep - - merged_file_paths.append(merged_file_path) - - merge(output_file_path, merged_file_paths, append_linesep) - - with open(output_file_path, "r") as output_file: - actual_output_file_content = output_file.read() - - assert actual_output_file_content == expected_output_file_content - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 3edb2451..90f68d02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,9 @@ target-version = 'py312' [tool.ruff.lint] select = [ - # pycodestyle - "E", + "E", # pycodestyle + "F", # pyflakes + "UP", # pyupgrade ] # ==== isort ==== diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index 30288520..093bfec4 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -42,7 +42,7 @@ def update_groups(self, user, claims): def create_user(self, claims): """Wrap the superclass's user creation.""" - user = super(ESnetAuthBackend, self).create_user(claims) + user = super().create_user(claims) return self.update_user(user, claims) def update_user(self, user, claims): diff --git a/scram/users/tests/factories.py b/scram/users/tests/factories.py index 1eca95c0..6db50f84 100644 --- a/scram/users/tests/factories.py +++ b/scram/users/tests/factories.py @@ -1,6 +1,7 @@ """Define Factory tests for the Users application.""" -from typing import Any, Sequence +from typing import Any +from collections.abc import Sequence from django.contrib.auth import get_user_model from factory import Faker, post_generation diff --git a/translator/gobgp.py b/translator/gobgp.py index 6c10e159..a50a1ea4 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -19,7 +19,7 @@ logging.basicConfig(level=logging.DEBUG) -class GoBGP(object): +class GoBGP: """Represents a GoBGP instance.""" def __init__(self, url): From 1d4f5580b2ec36c65315adbf1b3eb6f1cdd4cd85 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:50:46 -0600 Subject: [PATCH 090/156] Update manage.py to follow Django --- manage.py | 29 +++++++++----------- scram/route_manager/tests/test_websockets.py | 8 ++---- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/manage.py b/manage.py index 37181fe1..eec8580b 100755 --- a/manage.py +++ b/manage.py @@ -1,27 +1,21 @@ #!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" import os import sys from pathlib import Path -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") try: from django.core.management import execute_from_command_line - except ImportError: - # The above import may fail for some other reason. Ensure that the - # issue is really that Django is missing to avoid masking other - # exceptions on Python 2. - try: - import django # noqa - except ImportError: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) - - raise + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc # This allows easy placement of apps within the interior # scram directory. @@ -29,3 +23,6 @@ sys.path.append(str(current_path / "scram")) execute_from_command_line(sys.argv) + +if __name__ == "__main__": + main() diff --git a/scram/route_manager/tests/test_websockets.py b/scram/route_manager/tests/test_websockets.py index d8b58bc4..2a1b4ed5 100644 --- a/scram/route_manager/tests/test_websockets.py +++ b/scram/route_manager/tests/test_websockets.py @@ -26,15 +26,13 @@ async def get_communicators(actiontypes, should_match, *args, **kwds): Returns a list of (communicator, should_match bool) pairs. """ - assert len(actiontypes) == len(should_match) - router = URLRouter(websocket_urlpatterns) communicators = [ WebsocketCommunicator(router, f"/ws/route_manager/translator_{actiontype}/") for actiontype in actiontypes ] - response = zip(communicators, should_match) + response = zip(communicators, should_match, strict=True) - for communicator, should_match in response: + for communicator, _ in response: connected, _ = await communicator.connect() assert connected @@ -42,7 +40,7 @@ async def get_communicators(actiontypes, should_match, *args, **kwds): yield response finally: - for communicator, should_match in response: + for communicator, _ in response: await communicator.disconnect() From af564fd905fc8a4ac2a503ee5bdd9dd9c94abfe5 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:55:36 -0600 Subject: [PATCH 091/156] Add bugbear ruff checks --- config/consumers.py | 3 ++- pyproject.toml | 1 + scram/route_manager/api/views.py | 4 ++-- scram/route_manager/views.py | 6 ++++-- translator/gobgp.py | 4 +++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/config/consumers.py b/config/consumers.py index 11cca2ef..b7c4a618 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -2,6 +2,7 @@ from asgiref.sync import sync_to_async from channels.generic.websocket import AsyncJsonWebsocketConsumer +from functools import partial from scram.route_manager.models import Entry, WebSocketSequenceElement @@ -28,7 +29,7 @@ async def connect(self): for route in routes: for element in elements: - msg = await sync_to_async(lambda: element.websocketmessage)() + msg = await sync_to_async(partial(element.websocketmessage)) msg.msg_data[msg.msg_data_route_field] = str(route) await self.send_json({"type": msg.msg_type, "message": msg.msg_data}) diff --git a/pyproject.toml b/pyproject.toml index 90f68d02..93001d16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ target-version = 'py312' [tool.ruff.lint] select = [ + "B", # flake8-bugbear "E", # pycodestyle "F", # pyflakes "UP", # pyupgrade diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 46e12021..05ee47b3 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -166,13 +166,13 @@ def find_entries(arg, active_filter=None): try: pk = int(arg) query = Q(pk=pk) - except ValueError: + except ValueError as exc: # Maybe a CIDR? We want the ValueError at this point, if not. cidr = ipaddress.ip_network(arg, strict=False) min_prefix = getattr(settings, f"V{cidr.version}_MINPREFIX", 0) if cidr.prefixlen < min_prefix: - raise PrefixTooLarge() + raise PrefixTooLarge() from exc query = Q(route__route__net_overlaps=cidr) diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 99cc2c52..9f844901 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -24,8 +24,10 @@ channel_layer = get_channel_layer() -def home_page(request, prefilter=Entry.objects.all()): +def home_page(request, prefilter=None): """Return the home page, autocreating a user if none exists.""" + if not prefilter: + prefilter = Entry.objects.all() num_entries = settings.RECENT_LIMIT if request.user.has_perms(("route_manager.view_entry", "route_manager.add_entry")): readwrite = True @@ -116,7 +118,7 @@ def add_entry(request): for error in v: errors.append(f"'{k}' error: {str(error)}") else: - for k, v in res.data.items(): + for v in res.data.values(): errors.append(f"error: {str(v)}") messages.add_message(request, messages.ERROR, "
".join(errors)) elif res.status_code == 403: diff --git a/translator/gobgp.py b/translator/gobgp.py index a50a1ea4..20f3db76 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -33,8 +33,10 @@ def _get_family_AFI(self, ip_version): else: return gobgp_pb2.Family.AFI_IP - def _build_path(self, ip, event_data={}): + def _build_path(self, ip, event_data=None): # Grab ASN and Community from our event_data, or use the defaults + if not event_data: + event_data = {} asn = event_data.get("asn", DEFAULT_ASN) community = event_data.get("community", DEFAULT_COMMUNITY) ip_version = ip.ip.version From f240c7729f0ddf120a7418744d4cf916a659ee10 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 10:56:43 -0600 Subject: [PATCH 092/156] Format fix --- manage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manage.py b/manage.py index eec8580b..01c286e1 100755 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" + import os import sys from pathlib import Path @@ -24,5 +25,6 @@ def main(): execute_from_command_line(sys.argv) + if __name__ == "__main__": main() From 46fbee4e0dc2381b6b9cff8ab28c94677b1bea18 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:11:02 -0600 Subject: [PATCH 093/156] Add simplify ruff checks --- pyproject.toml | 5 +++++ translator/tests/acceptance/steps/actions.py | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 93001d16..fbf1eb37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,8 +40,13 @@ select = [ "B", # flake8-bugbear "E", # pycodestyle "F", # pyflakes + "SIM", # flake8-simplify "UP", # pyupgrade ] +ignore = [ + "SIM102", # Use a single `if` statement instead of nested `if` statements + "SIM108", # Use ternary operator instead of `if`-`else`-block +] # ==== isort ==== [tool.isort] diff --git a/translator/tests/acceptance/steps/actions.py b/translator/tests/acceptance/steps/actions.py index 1e03e8e1..83526b27 100644 --- a/translator/tests/acceptance/steps/actions.py +++ b/translator/tests/acceptance/steps/actions.py @@ -33,11 +33,7 @@ def get_block_status(context, ip): ip_obj = ipaddress.ip_interface(ip) - for path in context.gobgp.get_prefixes(ip_obj): - if ip_obj in ipaddress.ip_network(path.destination.prefix): - return True - - return False + return any(ip_obj in ipaddress.ip_network(path.destination.prefix) for path in context.gobgp.get_prefixes(ip_obj)) @capture From 4468747609b6329a6935a6d3f5de442f8f700576 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:11:58 -0600 Subject: [PATCH 094/156] Add isort ruff checks --- config/consumers.py | 2 +- scram/users/tests/factories.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/consumers.py b/config/consumers.py index b7c4a618..c1cd465a 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -1,8 +1,8 @@ import logging +from functools import partial from asgiref.sync import sync_to_async from channels.generic.websocket import AsyncJsonWebsocketConsumer -from functools import partial from scram.route_manager.models import Entry, WebSocketSequenceElement diff --git a/scram/users/tests/factories.py b/scram/users/tests/factories.py index 6db50f84..10331878 100644 --- a/scram/users/tests/factories.py +++ b/scram/users/tests/factories.py @@ -1,7 +1,7 @@ """Define Factory tests for the Users application.""" -from typing import Any from collections.abc import Sequence +from typing import Any from django.contrib.auth import get_user_model from factory import Faker, post_generation From 836223a2dc9e2b68a6886b2bd1b5e3f932320afc Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:31:31 -0600 Subject: [PATCH 095/156] Add docstyle ruff checks --- .ci-scripts/create_user.py | 11 ----------- config/__init__.py | 1 + config/api_router.py | 2 ++ config/asgi.py | 3 +-- config/consumers.py | 13 +++++++++++-- config/routing.py | 2 ++ config/settings/__init__.py | 1 + config/settings/base.py | 4 +--- config/settings/test.py | 4 +--- config/urls.py | 2 ++ config/websocket.py | 3 +++ config/wsgi.py | 3 +-- docs/__init__.py | 2 +- pyproject.toml | 2 ++ scram/contrib/__init__.py | 3 +-- scram/contrib/sites/__init__.py | 3 +-- scram/route_manager/api/serializers.py | 8 ++++---- scram/route_manager/api/views.py | 3 +-- scram/route_manager/tests/test_autocreate_admin.py | 2 +- scram/route_manager/tests/test_swagger.py | 2 +- scram/users/api/serializers.py | 4 ++-- scram/users/tests/test_forms.py | 9 ++++----- scram/users/tests/test_views.py | 6 +++--- translator/exceptions.py | 2 +- translator/shared.py | 6 +++--- translator/translator.py | 3 +-- 26 files changed, 52 insertions(+), 52 deletions(-) delete mode 100755 .ci-scripts/create_user.py diff --git a/.ci-scripts/create_user.py b/.ci-scripts/create_user.py deleted file mode 100755 index 1b263b8b..00000000 --- a/.ci-scripts/create_user.py +++ /dev/null @@ -1,11 +0,0 @@ -import django - -django.setup() - -from scram.users.models import User # noqa:E402 - -u, created = User.objects.get_or_create(username="admin") -u.set_password("password") -u.is_staff = True -u.is_superuser = True -u.save() diff --git a/config/__init__.py b/config/__init__.py index e69de29b..69052b7a 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -0,0 +1 @@ +"""Holds the setings and entrypoints.""" diff --git a/config/api_router.py b/config/api_router.py index a5b77f57..a57a12ba 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -1,3 +1,5 @@ +"""Map the API routes to the views.""" + from rest_framework.routers import DefaultRouter from scram.route_manager.api.views import ActionTypeViewSet, ClientViewSet, EntryViewSet, IgnoreEntryViewSet diff --git a/config/asgi.py b/config/asgi.py index c261d139..8064b2f2 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -1,5 +1,4 @@ -""" -ASGI config for SCRAM project. +"""ASGI config for SCRAM project. It exposes the ASGI callable as a module-level variable named ``application``. diff --git a/config/consumers.py b/config/consumers.py index c1cd465a..dc477e47 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -1,3 +1,5 @@ +"""Define logic for the WebSocket consumers.""" + import logging from functools import partial @@ -8,7 +10,10 @@ class TranslatorConsumer(AsyncJsonWebsocketConsumer): + """Handle messages from the Translator(s).""" + async def connect(self): + """Handle the initial connection with adding to the right group.""" logging.info("Translator connected") self.actiontype = self.scope["url_route"]["kwargs"]["actiontype"] self.translator_group = f"translator_{self.actiontype}" @@ -34,11 +39,12 @@ async def connect(self): await self.send_json({"type": msg.msg_type, "message": msg.msg_data}) async def disconnect(self, close_code): + """Discard any remaining messages on disconnect.""" logging.info(f"Disconnect received: {close_code}") await self.channel_layer.group_discard(self.translator_group, self.channel_name) async def receive_json(self, content): - """Received a WebSocket message""" + """Handle a WebSocket message.""" if content["type"] == "translator_check_resp": # We received a check response from a translator, forward to web UI. channel = content.pop("channel") @@ -59,14 +65,17 @@ async def _send_event(self, event): class WebUIConsumer(AsyncJsonWebsocketConsumer): + """Handle messages from the Web UI.""" + async def connect(self): + """Handle the initial connection with adding to the right group.""" self.actiontype = self.scope["url_route"]["kwargs"]["actiontype"] self.translator_group = f"translator_{self.actiontype}" await self.accept() - # Receive message from WebSocket async def receive_json(self, content): + """Receive message from WebSocket.""" if content["type"] == "wui_check_req": # Web UI asks us to check; forward to translator(s) await self.channel_layer.group_send( diff --git a/config/routing.py b/config/routing.py index bae0298a..10a63105 100644 --- a/config/routing.py +++ b/config/routing.py @@ -1,3 +1,5 @@ +"""Define URLs for the WebSocket consumers.""" + from django.urls import re_path from . import consumers diff --git a/config/settings/__init__.py b/config/settings/__init__.py index e69de29b..dd91545c 100644 --- a/config/settings/__init__.py +++ b/config/settings/__init__.py @@ -0,0 +1 @@ +"""Define Django settings. Everyone gets base, and then we can override in local or production.""" diff --git a/config/settings/base.py b/config/settings/base.py index edaa90f9..12d2378c 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -1,6 +1,4 @@ -""" -Base settings to build other settings files upon. -""" +"""Base settings to build other settings files upon.""" import logging import os diff --git a/config/settings/test.py b/config/settings/test.py index a4951b2c..8ad90c70 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -1,6 +1,4 @@ -""" -With these settings, tests run faster. -""" +"""With these settings, tests run faster.""" from .base import * # noqa from .base import env diff --git a/config/urls.py b/config/urls.py index 0a4648b7..a10e0dbd 100644 --- a/config/urls.py +++ b/config/urls.py @@ -1,3 +1,5 @@ +"""Define the non-WebSocket URLs for Django.""" + from django.conf import settings from django.conf.urls.static import static from django.contrib import admin diff --git a/config/websocket.py b/config/websocket.py index 81adfbc6..c81e93c3 100644 --- a/config/websocket.py +++ b/config/websocket.py @@ -1,4 +1,7 @@ +"""TODO: Find out if this is used.""" + async def websocket_application(scope, receive, send): + """Handle WebSocket messages. I guess.""" while True: event = await receive() diff --git a/config/wsgi.py b/config/wsgi.py index 37c0e92e..b679ee04 100644 --- a/config/wsgi.py +++ b/config/wsgi.py @@ -1,5 +1,4 @@ -""" -WSGI config for SCRAM project. +"""WSGI config for SCRAM project. This module contains the WSGI application used by Django's development server and any production WSGI deployments. It should expose a module-level variable diff --git a/docs/__init__.py b/docs/__init__.py index 8772c827..d34efb05 100644 --- a/docs/__init__.py +++ b/docs/__init__.py @@ -1 +1 @@ -# Included so that Django's startproject comment runs against the docs directory +"""Included so that Django's startproject comment runs against the docs directory.""" diff --git a/pyproject.toml b/pyproject.toml index fbf1eb37..7aec8aab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,8 +38,10 @@ target-version = 'py312' [tool.ruff.lint] select = [ "B", # flake8-bugbear + "D", # pydocstyle "E", # pycodestyle "F", # pyflakes + "I", # isort "SIM", # flake8-simplify "UP", # pyupgrade ] diff --git a/scram/contrib/__init__.py b/scram/contrib/__init__.py index 88d92bd3..adb76b31 100644 --- a/scram/contrib/__init__.py +++ b/scram/contrib/__init__.py @@ -1,5 +1,4 @@ -""" -To understand why this file is here, please read the CookieCutter documentation. +"""To understand why this file is here, please read the CookieCutter documentation. http://cookiecutter-django.readthedocs.io/en/latest/faq.html#why-is-there-a-django-contrib-sites-directory-in-cookiecutter-django """ diff --git a/scram/contrib/sites/__init__.py b/scram/contrib/sites/__init__.py index 88d92bd3..adb76b31 100644 --- a/scram/contrib/sites/__init__.py +++ b/scram/contrib/sites/__init__.py @@ -1,5 +1,4 @@ -""" -To understand why this file is here, please read the CookieCutter documentation. +"""To understand why this file is here, please read the CookieCutter documentation. http://cookiecutter-django.readthedocs.io/en/latest/faq.html#why-is-there-a-django-contrib-sites-directory-in-cookiecutter-django """ diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index ba3cae4b..ec6aa1b0 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -15,13 +15,13 @@ @extend_schema_field(field={"type": "string", "format": "cidr"}) class CustomCidrAddressField(rest_framework.CidrAddressField): - """This serializer defines a wrapper field so swagger can properly handle the inherited field.""" + """Define a wrapper field so swagger can properly handle the inherited field.""" pass class ActionTypeSerializer(serializers.ModelSerializer): - """This serializer defines no new fields.""" + """Map the serializer to the model via Meta.""" class Meta: """Maps to the ActionType model, and specifies the fields exposed by the API.""" @@ -45,7 +45,7 @@ class Meta: class ClientSerializer(serializers.ModelSerializer): - """This serializer defines no new fields.""" + """Map the serializer to the model via Meta.""" class Meta: """Maps to the Client model, and specifies the fields exposed by the API.""" @@ -96,7 +96,7 @@ def create(self, validated_data): class IgnoreEntrySerializer(serializers.ModelSerializer): - """This serializer defines no new fields.""" + """Map the route to the right field type.""" route = CustomCidrAddressField() diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 05ee47b3..c94f1da3 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -77,8 +77,7 @@ class EntryViewSet(viewsets.ModelViewSet): http_method_names = ["get", "post", "head", "delete"] def get_permissions(self): - """ - Override the permissions classes for POST method since we want to accept Entry creates from any client. + """Override the permissions classes for POST method since we want to accept Entry creates from any client. Note: We make authorization decisions on whether to actually create the object in the perform_create method later. diff --git a/scram/route_manager/tests/test_autocreate_admin.py b/scram/route_manager/tests/test_autocreate_admin.py index a550e3cb..7a83094f 100644 --- a/scram/route_manager/tests/test_autocreate_admin.py +++ b/scram/route_manager/tests/test_autocreate_admin.py @@ -1,4 +1,4 @@ -"""This file contains tests for the auto-creation of an admin user.""" +"""Test the auto-creation of an admin user.""" import pytest from django.contrib.messages import get_messages diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py index fc1c0fe5..aae60e9b 100644 --- a/scram/route_manager/tests/test_swagger.py +++ b/scram/route_manager/tests/test_swagger.py @@ -1,4 +1,4 @@ -"""This file contains tests for the swagger API endpoints.""" +"""Test the swagger API endpoints.""" import pytest from django.urls import reverse diff --git a/scram/users/api/serializers.py b/scram/users/api/serializers.py index a49520d0..f8d22c88 100644 --- a/scram/users/api/serializers.py +++ b/scram/users/api/serializers.py @@ -7,10 +7,10 @@ class UserSerializer(serializers.ModelSerializer): - """This serializer defines no new fields.""" + """Map to the User model.""" class Meta: - """Maps to the User model, and specifies the fields exposed by the API.""" + """Specify the fields exposed by the API.""" model = User fields = ["username", "name", "url"] diff --git a/scram/users/tests/test_forms.py b/scram/users/tests/test_forms.py index 4ff89f37..4a03993b 100644 --- a/scram/users/tests/test_forms.py +++ b/scram/users/tests/test_forms.py @@ -13,12 +13,11 @@ class TestUserCreationForm: """Test class for all tests related to the UserCreationForm.""" def test_username_validation_error_msg(self, user: User): - """ - Tests UserCreation Form's unique validator functions correctly by testing 3 things. + """Tests UserCreation Form's unique validator functions correctly by testing 3 things. - 1) A new user with an existing username cannot be added. - 2) Only 1 error is raised by the UserCreation Form - 3) The desired error message is raised + 1) A new user with an existing username cannot be added. + 2) Only 1 error is raised by the UserCreation Form + 3) The desired error message is raised """ # The user already exists, # hence cannot be created. diff --git a/scram/users/tests/test_views.py b/scram/users/tests/test_views.py index 63ad97bc..2df1929a 100644 --- a/scram/users/tests/test_views.py +++ b/scram/users/tests/test_views.py @@ -18,14 +18,14 @@ class TestUserUpdateView: - """ - Define tests related to the Update View. + """Define tests related to the Update View. - TODO: + Todo: extracting view initialization code as class-scoped fixture would be great if only pytest-django supported non-function-scoped fixture db access -- this is a work-in-progress for now: https://github.com/pytest-dev/pytest-django/pull/258 + """ def test_get_success_url(self, user: User, rf: RequestFactory): diff --git a/translator/exceptions.py b/translator/exceptions.py index a999ec8c..b6d2cfd6 100644 --- a/translator/exceptions.py +++ b/translator/exceptions.py @@ -1,4 +1,4 @@ -"""This module holds all of the exceptions we want to raise in our translators.""" +"""Define all of the exceptions we want to raise in our translators.""" class ASNError(TypeError): diff --git a/translator/shared.py b/translator/shared.py index 27e89f97..fa128521 100644 --- a/translator/shared.py +++ b/translator/shared.py @@ -1,11 +1,10 @@ -"""This module provides a location for code that we want to share between all translators.""" +"""Provide a location for code that we want to share between all translators.""" from exceptions import ASNError def asn_is_valid(asn: int) -> bool: - """ - asn_is_valid makes sure that an ASN passed in is a valid 2 or 4 Byte ASN. + """asn_is_valid makes sure that an ASN passed in is a valid 2 or 4 Byte ASN. Args: asn (int): The Autonomous System Number that we want to validate @@ -15,6 +14,7 @@ def asn_is_valid(asn: int) -> bool: Returns: bool: _description_ + """ if not isinstance(asn, int): raise ASNError(f"ASN {asn} is not an Integer, has type {type(asn)}") diff --git a/translator/translator.py b/translator/translator.py index b5a7dd02..7e1a6c64 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -16,8 +16,7 @@ if debug_mode: def install_deps(): - """ - Install necessary dependencies for debuggers. + """Install necessary dependencies for debuggers. Because of how we build translator currently, we don't have a great way to selectively install things at build, so we just do it here! Right now this also includes base.txt, From 4e2691476a32f9fae6166e99e681d765c8e8d2e1 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:32:17 -0600 Subject: [PATCH 096/156] Format fix --- config/websocket.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config/websocket.py b/config/websocket.py index c81e93c3..b48dfb8b 100644 --- a/config/websocket.py +++ b/config/websocket.py @@ -1,5 +1,6 @@ """TODO: Find out if this is used.""" + async def websocket_application(scope, receive, send): """Handle WebSocket messages. I guess.""" while True: From bec13b9bffa57cabf2283a51722dd80beaa186eb Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:35:55 -0600 Subject: [PATCH 097/156] Add pep8-naming ruff checks --- pyproject.toml | 1 + scram/route_manager/tests/test_websockets.py | 10 +++++----- translator/gobgp.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7aec8aab..47bf3698 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ select = [ "E", # pycodestyle "F", # pyflakes "I", # isort + "N", # pep8-naming "SIM", # flake8-simplify "UP", # pyupgrade ] diff --git a/scram/route_manager/tests/test_websockets.py b/scram/route_manager/tests/test_websockets.py index 2a1b4ed5..18b64d09 100644 --- a/scram/route_manager/tests/test_websockets.py +++ b/scram/route_manager/tests/test_websockets.py @@ -78,9 +78,9 @@ def setUp(self): self.generate_add_msgs = [lambda ip, mask: {"type": "translator_add", "message": {"route": f"{ip}/{mask}"}}] # Now we run any local setup actions by the child classes - self.local_setUp() + self.local_setup() - def local_setUp(self): + def local_setup(self): """Allow child classes to override this if desired.""" return @@ -151,7 +151,7 @@ async def test_add_v6(self): class TranslatorDontCrossTheStreamsTestCase(TestTranslatorBaseCase): """Two translators in one group, two in another group, single IP, ensure we get only the messages we expect.""" - def local_setUp(self): + def local_setup(self): """Define the actions and what we expect.""" self.actiontypes = ["block", "block", "noop", "noop"] self.should_match = [True, True, False, False] @@ -160,7 +160,7 @@ def local_setUp(self): class TranslatorSequenceTestCase(TestTranslatorBaseCase): """Test a sequence of WebSocket messages.""" - def local_setUp(self): + def local_setup(self): """Define the messages we want to send.""" wsm2 = WebSocketMessage.objects.create(msg_type="translator_add", msg_data_route_field="foo") _ = WebSocketSequenceElement.objects.create( @@ -181,7 +181,7 @@ def local_setUp(self): class TranslatorParametersTestCase(TestTranslatorBaseCase): """Additional parameters in the JSONField.""" - def local_setUp(self): + def local_setup(self): """Define the message we want to send.""" wsm = WebSocketMessage.objects.get(msg_type="translator_add", msg_data_route_field="route") wsm.msg_data = {"asn": 65550, "community": 100, "route": "Ensure this gets overwritten."} diff --git a/translator/gobgp.py b/translator/gobgp.py index 20f3db76..1192f1c6 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -27,7 +27,7 @@ def __init__(self, url): channel = grpc.insecure_channel(url) self.stub = gobgp_pb2_grpc.GobgpApiStub(channel) - def _get_family_AFI(self, ip_version): + def _get_family_afi(self, ip_version): if ip_version == 6: return gobgp_pb2.Family.AFI_IP6 else: @@ -63,7 +63,7 @@ def _build_path(self, ip, event_data=None): # Set the next hop to the correct value depending on IP family next_hop = Any() - family_afi = self._get_family_AFI(ip_version) + family_afi = self._get_family_afi(ip_version) if ip_version == 6: next_hops = event_data.get("next_hop", DEFAULT_V6_NEXTHOP) next_hop.Pack( @@ -156,7 +156,7 @@ def del_path(self, ip, event_data): def get_prefixes(self, ip): """Retrieve the routes that match a prefix and are announced.""" prefixes = [gobgp_pb2.TableLookupPrefix(prefix=str(ip.ip))] - family_afi = self._get_family_AFI(ip.ip.version) + family_afi = self._get_family_afi(ip.ip.version) result = self.stub.ListPath( gobgp_pb2.ListPathRequest( table_type=gobgp_pb2.GLOBAL, From 60761496a372de45fddef010bcb93f600aa9d20b Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:49:54 -0600 Subject: [PATCH 098/156] Add bandit ruff security checks --- config/asgi.py | 2 +- config/settings/local.py | 2 +- pyproject.toml | 11 +++++++++++ translator/translator.py | 4 ++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/config/asgi.py b/config/asgi.py index 8064b2f2..b42b7444 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -35,7 +35,7 @@ import debugpy - debugpy.listen(("0.0.0.0", 56780)) + debugpy.listen(("0.0.0.0", 56780)) # noqa S104 (doesn't like binding to all interfaces) logging.info("Debugger listening on port 56780.") else: diff --git a/config/settings/local.py b/config/settings/local.py index 8d81f04d..f68d278b 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -11,7 +11,7 @@ default="BmZnn8FeNFdaeCod8ky6eBNpTiwO45NzlFyA6kk1xo0g4Mc263gAyscHFCMCeJAi", ) # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts -ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1", "django"] +ALLOWED_HOSTS = ["*"] # CACHES # ------------------------------------------------------------------------------ diff --git a/pyproject.toml b/pyproject.toml index 47bf3698..44ffd107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,12 +37,14 @@ target-version = 'py312' [tool.ruff.lint] select = [ + "ASYNC", # flake8-async "B", # flake8-bugbear "D", # pydocstyle "E", # pycodestyle "F", # pyflakes "I", # isort "N", # pep8-naming + "S", # flake8-bandit "SIM", # flake8-simplify "UP", # pyupgrade ] @@ -51,6 +53,15 @@ ignore = [ "SIM108", # Use ternary operator instead of `if`-`else`-block ] +[tool.ruff.lint.per-file-ignores] +"**/{tests}/*" = [ + "S101", # use of assert + "S106", # hardcoded password +] +"test.py" = [ + "S105", # hardcoded password as argument +] + # ==== isort ==== [tool.isort] profile = "black" diff --git a/translator/translator.py b/translator/translator.py index 7e1a6c64..7a2b964e 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -28,7 +28,7 @@ def install_deps(): import subprocess import sys - subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "/requirements/local.txt"]) + subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "/requirements/local.txt"]) # noqa: S603 TODO: add this to the container build logging.info("Done installing dependencies for debuggers") @@ -52,7 +52,7 @@ def install_deps(): import debugpy - debugpy.listen(("0.0.0.0", 56781)) + debugpy.listen(("0.0.0.0", 56781)) # noqa S104 (doesn't like binding to all interfaces) logging.info("Debugger listening on port 56781.") else: From 03d765d332422f3309fd8054ad1fa0cbb5365e2d Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:50:14 -0600 Subject: [PATCH 099/156] Format fix --- config/asgi.py | 2 +- translator/translator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/asgi.py b/config/asgi.py index b42b7444..4fda9441 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -35,7 +35,7 @@ import debugpy - debugpy.listen(("0.0.0.0", 56780)) # noqa S104 (doesn't like binding to all interfaces) + debugpy.listen(("0.0.0.0", 56780)) # noqa S104 (doesn't like binding to all interfaces) logging.info("Debugger listening on port 56780.") else: diff --git a/translator/translator.py b/translator/translator.py index 7a2b964e..abc8eab2 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -28,7 +28,7 @@ def install_deps(): import subprocess import sys - subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "/requirements/local.txt"]) # noqa: S603 TODO: add this to the container build + subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "/requirements/local.txt"]) # noqa: S603 TODO: add this to the container build logging.info("Done installing dependencies for debuggers") @@ -52,7 +52,7 @@ def install_deps(): import debugpy - debugpy.listen(("0.0.0.0", 56781)) # noqa S104 (doesn't like binding to all interfaces) + debugpy.listen(("0.0.0.0", 56781)) # noqa S104 (doesn't like binding to all interfaces) logging.info("Debugger listening on port 56781.") else: From 97e876c0062270d762f8659f18c99f0562b2a5e2 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 11:59:12 -0600 Subject: [PATCH 100/156] Add boolean-trap ruff checks --- pyproject.toml | 5 +++++ scram/route_manager/tests/acceptance/steps/translator.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 44ffd107..90e5b537 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,9 +39,11 @@ target-version = 'py312' select = [ "ASYNC", # flake8-async "B", # flake8-bugbear + "BLE", # flake8-blind-except "D", # pydocstyle "E", # pycodestyle "F", # pyflakes + "FBT", # boolean-trap "I", # isort "N", # pep8-naming "S", # flake8-bandit @@ -61,6 +63,9 @@ ignore = [ "test.py" = [ "S105", # hardcoded password as argument ] +"factories.py" = [ + "FBT001", # minimal issue; don't need to mess with in the User app +] # ==== isort ==== [tool.isort] diff --git a/scram/route_manager/tests/acceptance/steps/translator.py b/scram/route_manager/tests/acceptance/steps/translator.py index 9fdecb3a..adc4812d 100644 --- a/scram/route_manager/tests/acceptance/steps/translator.py +++ b/scram/route_manager/tests/acceptance/steps/translator.py @@ -7,7 +7,7 @@ from config.asgi import ws_application -async def query_translator(route, actiontype, is_announced=True): +async def query_translator(route, actiontype, is_announced): """Ensure the specified route is currently either blocked or unblocked.""" communicator = WebsocketCommunicator(ws_application, f"/ws/route_manager/webui_{actiontype}/") connected, subprotocol = await communicator.connect() @@ -25,7 +25,7 @@ async def query_translator(route, actiontype, is_announced=True): @async_run_until_complete async def check_blocked(context, route, actiontype): """Ensure the specified route is currently blocked.""" - await query_translator(route, actiontype) + await query_translator(route, actiontype, is_announced=True) @then("{route} is not announced by {actiontype} translators") From 6dac72ad12108fd3e12c9b893e0c35032efd6473 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:02:49 -0600 Subject: [PATCH 101/156] Add builtins and commas checks --- config/asgi.py | 2 +- config/consumers.py | 2 +- config/settings/base.py | 6 +++--- config/settings/local.py | 2 +- config/settings/production.py | 6 +++--- config/settings/test.py | 2 +- manage.py | 2 +- pyproject.toml | 2 ++ scram/route_manager/api/serializers.py | 4 +++- scram/route_manager/api/views.py | 6 ++++-- .../route_manager/tests/acceptance/steps/common.py | 6 ++++-- scram/route_manager/tests/test_websockets.py | 14 +++++++++++--- scram/route_manager/views.py | 2 +- scram/users/tests/test_forms.py | 2 +- translator/gobgp.py | 8 ++++---- 15 files changed, 41 insertions(+), 25 deletions(-) diff --git a/config/asgi.py b/config/asgi.py index 4fda9441..28a83655 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -62,5 +62,5 @@ { "http": django_application, "websocket": ws_application, - } + }, ) diff --git a/config/consumers.py b/config/consumers.py index dc477e47..e0aede04 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -23,7 +23,7 @@ async def connect(self): # Filter WebSocketSequenceElements by actiontype elements = await sync_to_async(list)( - WebSocketSequenceElement.objects.filter(action_type__name=self.actiontype).order_by("order_num") + WebSocketSequenceElement.objects.filter(action_type__name=self.actiontype).order_by("order_num"), ) if not elements: logging.warning(f"No elements found for actiontype={self.actiontype}.") diff --git a/config/settings/base.py b/config/settings/base.py index 12d2378c..4927f868 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -186,7 +186,7 @@ "scram.route_manager.context_processors.login_logout", ], }, - } + }, ] # https://docs.djangoproject.com/en/dev/ref/settings/#form-renderer @@ -236,14 +236,14 @@ "version": 1, "disable_existing_loggers": False, "formatters": { - "verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"} + "verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}, }, "handlers": { "console": { "level": "DEBUG", "class": "logging.StreamHandler", "formatter": "verbose", - } + }, }, "root": {"level": "INFO", "handlers": ["console"]}, } diff --git a/config/settings/local.py b/config/settings/local.py index f68d278b..8fff98c4 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -20,7 +20,7 @@ "default": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", "LOCATION": "", - } + }, } # EMAIL diff --git a/config/settings/production.py b/config/settings/production.py index 8f62b5d4..93993a39 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -28,7 +28,7 @@ # https://github.com/jazzband/django-redis#memcached-exceptions-behavior "IGNORE_EXCEPTIONS": True, }, - } + }, } # SECURITY @@ -68,7 +68,7 @@ "django.template.loaders.filesystem.Loader", "django.template.loaders.app_directories.Loader", ], - ) + ), ] # EMAIL @@ -108,7 +108,7 @@ "disable_existing_loggers": False, "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, "formatters": { - "verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"} + "verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}, }, "handlers": { "mail_admins": { diff --git a/config/settings/test.py b/config/settings/test.py index 8ad90c70..d7142c8b 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -27,7 +27,7 @@ "django.template.loaders.filesystem.Loader", "django.template.loaders.app_directories.Loader", ], - ) + ), ] # EMAIL diff --git a/manage.py b/manage.py index 01c286e1..934b0c56 100755 --- a/manage.py +++ b/manage.py @@ -15,7 +15,7 @@ def main(): raise ImportError( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" + "forget to activate a virtual environment?", ) from exc # This allows easy placement of apps within the interior diff --git a/pyproject.toml b/pyproject.toml index 90e5b537..89cd8032 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,9 +37,11 @@ target-version = 'py312' [tool.ruff.lint] select = [ + "A", # flake8-builtins "ASYNC", # flake8-async "B", # flake8-bugbear "BLE", # flake8-blind-except + "COM", # flake8-commas "D", # pydocstyle "E", # pycodestyle "F", # pyflakes diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index ec6aa1b0..9d449c50 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -58,7 +58,9 @@ class EntrySerializer(serializers.HyperlinkedModelSerializer): """Due to the use of ForeignKeys, this follows some relationships to make sense via the API.""" url = serializers.HyperlinkedIdentityField( - view_name="api:v1:entry-detail", lookup_url_kwarg="pk", lookup_field="route" + view_name="api:v1:entry-detail", + lookup_url_kwarg="pk", + lookup_field="route", ) route = CustomCidrAddressField() actiontype = serializers.CharField(default="block") diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index c94f1da3..61add66f 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -113,7 +113,8 @@ def perform_create(self, serializer): if self.request.data.get("uuid"): client_uuid = self.request.data["uuid"] authorized_actiontypes = Client.objects.filter(uuid=client_uuid).values_list( - "authorized_actiontypes__name", flat=True + "authorized_actiontypes__name", + flat=True, ) authorized_client = Client.objects.filter(uuid=client_uuid).values("is_authorized") if not authorized_client or actiontype not in authorized_actiontypes: @@ -141,7 +142,8 @@ def perform_create(self, serializer): msg.msg_data[msg.msg_data_route_field] = str(route) # Must match a channel name defined in asgi.py async_to_sync(channel_layer.group_send)( - f"translator_{actiontype}", {"type": msg.msg_type, "message": msg.msg_data} + f"translator_{actiontype}", + {"type": msg.msg_type, "message": msg.msg_data}, ) serializer.save() diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index cc732b15..104edb91 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -17,7 +17,8 @@ def create_actiontype(context, name): """Create an actiontype of that name.""" context.channel_layer = get_channel_layer() async_to_sync(context.channel_layer.group_send)( - f"translator_{name}", {"type": "translator_remove_all", "message": {}} + f"translator_{name}", + {"type": "translator_remove_all", "message": {}}, ) at, created = ActionType.objects.get_or_create(name=name) @@ -154,7 +155,8 @@ def remove_expired(context): def add_ignore_entry(context, value): """Add an IgnoreEntry with the specified route.""" context.response = context.test.client.post( - reverse("api:v1:ignoreentry-list"), {"route": value, "comment": "test api"} + reverse("api:v1:ignoreentry-list"), + {"route": value, "comment": "test api"}, ) diff --git a/scram/route_manager/tests/test_websockets.py b/scram/route_manager/tests/test_websockets.py index 18b64d09..ca9cd528 100644 --- a/scram/route_manager/tests/test_websockets.py +++ b/scram/route_manager/tests/test_websockets.py @@ -69,7 +69,9 @@ def setUp(self): wsm, _ = WebSocketMessage.objects.get_or_create(msg_type="translator_add", msg_data_route_field="route") _, _ = WebSocketSequenceElement.objects.get_or_create( - websocketmessage=wsm, verb="A", action_type=self.actiontype + websocketmessage=wsm, + verb="A", + action_type=self.actiontype, ) # Set some defaults; some child classes override this @@ -164,11 +166,17 @@ def local_setup(self): """Define the messages we want to send.""" wsm2 = WebSocketMessage.objects.create(msg_type="translator_add", msg_data_route_field="foo") _ = WebSocketSequenceElement.objects.create( - websocketmessage=wsm2, verb="A", action_type=self.actiontype, order_num=20 + websocketmessage=wsm2, + verb="A", + action_type=self.actiontype, + order_num=20, ) wsm3 = WebSocketMessage.objects.create(msg_type="translator_add", msg_data_route_field="bar") _ = WebSocketSequenceElement.objects.create( - websocketmessage=wsm3, verb="A", action_type=self.actiontype, order_num=2 + websocketmessage=wsm3, + verb="A", + action_type=self.actiontype, + order_num=2, ) self.generate_add_msgs = [ diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 9f844901..827aabba 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -144,7 +144,7 @@ def process_expired(request): { "entries_deleted": entries_start - entries_end, "active_entries": entries_end, - } + }, ), content_type="application/json", ) diff --git a/scram/users/tests/test_forms.py b/scram/users/tests/test_forms.py index 4a03993b..3fe65da4 100644 --- a/scram/users/tests/test_forms.py +++ b/scram/users/tests/test_forms.py @@ -26,7 +26,7 @@ def test_username_validation_error_msg(self, user: User): "username": user.username, "password1": user.password, "password2": user.password, - } + }, ) assert not form.is_valid() diff --git a/translator/gobgp.py b/translator/gobgp.py index 1192f1c6..5cd17ac4 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -49,7 +49,7 @@ def _build_path(self, ip, event_data=None): origin.Pack( attribute_pb2.OriginAttribute( origin=2, - ) + ), ) # IP prefix and its associated length @@ -58,7 +58,7 @@ def _build_path(self, ip, event_data=None): attribute_pb2.IPAddressPrefix( prefix_len=ip.network.prefixlen, prefix=str(ip.ip), - ) + ), ) # Set the next hop to the correct value depending on IP family @@ -71,14 +71,14 @@ def _build_path(self, ip, event_data=None): family=gobgp_pb2.Family(afi=family_afi, safi=gobgp_pb2.Family.SAFI_UNICAST), next_hops=[next_hops], nlris=[nlri], - ) + ), ) else: next_hops = event_data.get("next_hop", DEFAULT_V4_NEXTHOP) next_hop.Pack( attribute_pb2.NextHopAttribute( next_hop=next_hops, - ) + ), ) # Set our AS Path From 5e614fe5d866c9da551a6693716c3fed4acc3616 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:05:26 -0600 Subject: [PATCH 102/156] Add comprehensions and datetimez checks --- pyproject.toml | 2 ++ scram/route_manager/tests/acceptance/steps/common.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 89cd8032..7a47b91c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,10 @@ select = [ "ASYNC", # flake8-async "B", # flake8-bugbear "BLE", # flake8-blind-except + "C4", # flake8-comprehensions "COM", # flake8-commas "D", # pydocstyle + "DTZ", # flake8-datetimez "E", # pycodestyle "F", # pyflakes "FBT", # boolean-trap diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index 104edb91..0b449e9f 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -1,4 +1,4 @@ -"""Define steps used exclusively by the Behave tests.""" +scram/route_manager/tests/acceptance/steps/common.py"""Define steps used exclusively by the Behave tests.""" import datetime import time @@ -123,7 +123,7 @@ def add_entry_with_absolute_expiration(context, value, exp): def add_entry_with_relative_expiration(context, value, secs): """Block the provided route and add a relative expiration.""" td = datetime.timedelta(seconds=secs) - expiration = datetime.datetime.now() + td + expiration = datetime.datetime.now(tz=datetime.UTC) + td context.response = context.test.client.post( reverse("api:v1:entry-list"), From 842c3eab2354592c51a7d70bcf2b258977a5c28e Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:06:36 -0600 Subject: [PATCH 103/156] Add comprehensions and datetimez checks --- scram/route_manager/tests/acceptance/steps/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index 0b449e9f..d4836fe9 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -1,4 +1,4 @@ -scram/route_manager/tests/acceptance/steps/common.py"""Define steps used exclusively by the Behave tests.""" +"""Define steps used exclusively by the Behave tests.""" import datetime import time From 65850f39899b6f9753ab898f03aaf9e7c430826f Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:20:20 -0600 Subject: [PATCH 104/156] Add Django and errmsg checks --- config/settings/base.py | 3 ++- config/settings/local.py | 3 ++- manage.py | 7 +++++-- pyproject.toml | 20 +++++++++--------- scram/route_manager/models.py | 38 +++++++++++++++++------------------ translator/shared.py | 6 ++++-- 6 files changed, 43 insertions(+), 34 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 4927f868..27584e5c 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -332,7 +332,8 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#logout-url LOGOUT_URL = "local_auth:logout" else: - raise ValueError(f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'") + msg = f"Invalid authentication method: {AUTH_METHOD}. Please choose 'local' or 'oidc'" + raise ValueError(msg) # Should we create an admin user for you diff --git a/config/settings/local.py b/config/settings/local.py index 8fff98c4..7b8ef9fc 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -73,7 +73,8 @@ # ------------------------------------------------------------------------------ # We shouldn't be using OIDC in local dev mode as of now, but might be worth pursuing later if AUTH_METHOD == "oidc": - raise NotImplementedError("oidc is not yet implemented") + msg = "oidc is not yet implemented" + raise NotImplementedError(msg) # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "admin:login" diff --git a/manage.py b/manage.py index 934b0c56..6f05512f 100755 --- a/manage.py +++ b/manage.py @@ -12,10 +12,13 @@ def main(): try: from django.core.management import execute_from_command_line except ImportError as exc: - raise ImportError( + msg = ( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?", + "forget to activate a virtual environment?" + ) + raise ImportError( + msg, ) from exc # This allows easy placement of apps within the interior diff --git a/pyproject.toml b/pyproject.toml index 7a47b91c..6fd74f21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,21 +37,23 @@ target-version = 'py312' [tool.ruff.lint] select = [ - "A", # flake8-builtins - "ASYNC", # flake8-async - "B", # flake8-bugbear - "BLE", # flake8-blind-except - "C4", # flake8-comprehensions - "COM", # flake8-commas + "A", # builtins + "ASYNC", # async + "B", # bugbear + "BLE", # blind-except + "C4", # comprehensions + "COM", # commas "D", # pydocstyle - "DTZ", # flake8-datetimez + "DJ", # django + "DTZ", # datetimez "E", # pycodestyle + "EM", # errmsg "F", # pyflakes "FBT", # boolean-trap "I", # isort "N", # pep8-naming - "S", # flake8-bandit - "SIM", # flake8-simplify + "S", # bandit + "SIM", # simplify "UP", # pyupgrade ] ignore = [ diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index 3bbf67e3..daf9cabe 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -17,14 +17,14 @@ class Route(models.Model): route = CidrAddressField(unique=True) uuid = models.UUIDField(db_index=True, default=uuid_lib.uuid4, editable=False) - def get_absolute_url(self): - """Ensure we use UUID on the API side instead.""" - return reverse("") - def __str__(self): """Don't display the UUID, only the route.""" return str(self.route) + def get_absolute_url(self): + """Ensure we use UUID on the API side instead.""" + return reverse("") + class ActionType(models.Model): """Define a type of action that can be done with a given route. e.g. Block, shunt, redirect, etc.""" @@ -88,7 +88,7 @@ class Entry(models.Model): route = models.ForeignKey("Route", on_delete=models.PROTECT) actiontype = models.ForeignKey("ActionType", on_delete=models.PROTECT) - comment = models.TextField(blank=True, null=True) + comment = models.TextField(blank=True, default="") is_active = models.BooleanField(default=True) # TODO: fix name if this works history = HistoricalRecords() @@ -98,10 +98,23 @@ class Entry(models.Model): expiration_reason = models.CharField( help_text="Optional reason for the expiration", max_length=200, - null=True, blank=True, + default="", ) + class Meta: + """Ensure that multiple routes can be added as long as they have different action types.""" + + unique_together = ["route", "actiontype"] + verbose_name_plural = "Entries" + + def __str__(self): + """Summarize the most important fields to something easily readable.""" + desc = f"{self.route} ({self.actiontype})" + if not self.is_active: + desc += " (inactive)" + return desc + def delete(self, *args, **kwargs): """Set inactive instead of deleting, as we want to ensure a history of entries.""" if not self.is_active: @@ -122,19 +135,6 @@ def delete(self, *args, **kwargs): }, ) - class Meta: - """Ensure that multiple routes can be added as long as they have different action types.""" - - unique_together = ["route", "actiontype"] - verbose_name_plural = "Entries" - - def __str__(self): - """Summarize the most important fields to something easily readable.""" - desc = f"{self.route} ({self.actiontype})" - if not self.is_active: - desc += " (inactive)" - return desc - def get_change_reason(self): """Traverse come complex relationships to determine the most recent change reason.""" hist_mgr = getattr(self, self._meta.simple_history_manager_attribute) diff --git a/translator/shared.py b/translator/shared.py index fa128521..ebe56e6d 100644 --- a/translator/shared.py +++ b/translator/shared.py @@ -17,9 +17,11 @@ def asn_is_valid(asn: int) -> bool: """ if not isinstance(asn, int): - raise ASNError(f"ASN {asn} is not an Integer, has type {type(asn)}") + msg = f"ASN {asn} is not an Integer, has type {type(asn)}" + raise ASNError(msg) if not 0 < asn < 4294967295: # This is the max as stated in rfc6996 - raise ASNError(f"ASN {asn} is out of range. Must be between 0 and 4294967295") + msg = f"ASN {asn} is out of range. Must be between 0 and 4294967295" + raise ASNError(msg) return True From 59d0ae10c98f10d3ab02339710d3fb0c5b4c0c34 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:22:29 -0600 Subject: [PATCH 105/156] Add implicit-str-concat checks --- config/settings/base.py | 2 +- config/settings/production.py | 2 +- pyproject.toml | 1 + scram/route_manager/models.py | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 27584e5c..caf76e65 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -236,7 +236,7 @@ "version": 1, "disable_existing_loggers": False, "formatters": { - "verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}, + "verbose": {"format": "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s"}, }, "handlers": { "console": { diff --git a/config/settings/production.py b/config/settings/production.py index 93993a39..cdb213f0 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -108,7 +108,7 @@ "disable_existing_loggers": False, "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, "formatters": { - "verbose": {"format": "%(levelname)s %(asctime)s %(module)s " "%(process)d %(thread)d %(message)s"}, + "verbose": {"format": "%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s"}, }, "handlers": { "mail_admins": { diff --git a/pyproject.toml b/pyproject.toml index 6fd74f21..9765526f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ select = [ "F", # pyflakes "FBT", # boolean-trap "I", # isort + "ISC", # implicit-str-concat "N", # pep8-naming "S", # bandit "SIM", # simplify diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index daf9cabe..37c37698 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -62,7 +62,7 @@ class WebSocketSequenceElement(models.Model): websocketmessage = models.ForeignKey("WebSocketMessage", on_delete=models.CASCADE) order_num = models.SmallIntegerField( "Sequences are sent from the smallest order_num to the highest. " - + "Messages with the same order_num could be sent in any order", + "Messages with the same order_num could be sent in any order", default=0, ) @@ -79,7 +79,7 @@ def __str__(self): """Summarize the fields into something short and readable.""" return ( f"{self.websocketmessage} as order={self.order_num} for " - + f"{self.verb} actions on actiontype={self.action_type}" + f"{self.verb} actions on actiontype={self.action_type}" ) From c473082d7f115da2dbcb120a3c35d6cd05d8f5a3 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:36:22 -0600 Subject: [PATCH 106/156] Add logging-format checks --- config/asgi.py | 4 ++-- config/consumers.py | 4 ++-- config/settings/base.py | 2 +- pyproject.toml | 3 +++ scram/route_manager/api/serializers.py | 2 +- scram/route_manager/api/views.py | 12 ++++++------ scram/route_manager/models.py | 2 +- translator/gobgp.py | 10 +++++----- translator/translator.py | 8 ++++---- 9 files changed, 25 insertions(+), 22 deletions(-) diff --git a/config/asgi.py b/config/asgi.py index 28a83655..6e6b8979 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -21,7 +21,7 @@ # Here we setup a debugger if this is desired. This obviously should not be run in production. debug = os.environ.get("DEBUG") if debug: - logging.info(f"Django is set to use a debugger. Provided debug mode: {debug}") + logging.info("Django is set to use a debugger. Provided debug mode:", debug) if debug == "pycharm-pydevd": logging.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") @@ -39,7 +39,7 @@ logging.info("Debugger listening on port 56780.") else: - logging.warning(f"Invalid debug mode given: {debug}. Debugger not started") + logging.warning("Invalid debug mode given: %s.", "Debugger not started", extra=debug) # This allows easy placement of apps within the interior # scram directory. diff --git a/config/consumers.py b/config/consumers.py index e0aede04..082b31b2 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -26,7 +26,7 @@ async def connect(self): WebSocketSequenceElement.objects.filter(action_type__name=self.actiontype).order_by("order_num"), ) if not elements: - logging.warning(f"No elements found for actiontype={self.actiontype}.") + logging.warning("No elements found for actiontype=%s.", extra=self.actiontype) return # Avoid lazy evaluation @@ -40,7 +40,7 @@ async def connect(self): async def disconnect(self, close_code): """Discard any remaining messages on disconnect.""" - logging.info(f"Disconnect received: {close_code}") + logging.info("Disconnect received:", close_code) await self.channel_layer.group_discard(self.translator_group, self.channel_name) async def receive_json(self, content): diff --git a/config/settings/base.py b/config/settings/base.py index caf76e65..1861af43 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -308,7 +308,7 @@ ) OIDC_RP_SIGN_ALGO = "RS256" -logging.info(f"Using AUTH METHOD = {AUTH_METHOD}") +logging.info("Using AUTH METHOD =", AUTH_METHOD) if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 diff --git a/pyproject.toml b/pyproject.toml index 9765526f..50fea91c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,11 @@ select = [ "EM", # errmsg "F", # pyflakes "FBT", # boolean-trap + "G", # logging-format "I", # isort + "ICN", # import-conventions "ISC", # implicit-str-concat + "LOG", # logging "N", # pep8-naming "S", # bandit "SIM", # simplify diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index 9d449c50..455f8f30 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -91,7 +91,7 @@ def create(self, validated_data): actiontype_instance = ActionType.objects.get(name=actiontype) entry_instance, created = Entry.objects.get_or_create(route=route_instance, actiontype=actiontype_instance) - logger.debug(f"{comment}") + logger.debug("Created entry with comment:", comment) update_change_reason(entry_instance, comment) return entry_instance diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 61add66f..98f47273 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -102,7 +102,7 @@ def perform_create(self, serializer): try: expiration = parse_datetime(tmp_exp) # noqa: F841 except ValueError: - logging.info(f"Could not parse expiration DateTime string {tmp_exp!r}.") + logging.warning("Could not parse expiration DateTime string:", tmp_exp) # Make sure we put in an acceptable sized prefix min_prefix = getattr(settings, f"V{route.version}_MINPREFIX", 0) @@ -118,8 +118,8 @@ def perform_create(self, serializer): ) authorized_client = Client.objects.filter(uuid=client_uuid).values("is_authorized") if not authorized_client or actiontype not in authorized_actiontypes: - logging.debug(f"Client {client_uuid} actiontypes: {authorized_actiontypes}") - logging.info(f"{client_uuid} is not allowed to add an entry to the {actiontype} list") + logging.debug("Client", client_uuid, "actiontypes:", authorized_actiontypes) + logging.info(client_uuid, "is not allowed to add an entry to the", actiontype, "list") raise ActiontypeNotAllowed() elif not self.request.user.has_perm("route_manager.can_add_entry"): raise PermissionDenied() @@ -130,12 +130,12 @@ def perform_create(self, serializer): ignore_entries = [] for ignore_entry in overlapping_ignore.values(): ignore_entries.append(str(ignore_entry["route"])) - logging.info(f"Cannot proceed adding {route}. The ignore list contains {ignore_entries}") + logging.info("Cannot proceed adding", route, " The ignore list contains", ignore_entries) raise IgnoredRoute else: elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") if not elements: - logging.warning(f"No elements found for actiontype={actiontype}.") + logging.warning("No elements found for actiontype:", actiontype) for element in elements: msg = element.websocketmessage @@ -154,7 +154,7 @@ def perform_create(self, serializer): entry.who = who entry.is_active = True entry.comment = comment - logging.info(f"{entry}") + logging.info("Created entry:", entry) entry.save() @staticmethod diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index 37c37698..35399820 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -122,7 +122,7 @@ def delete(self, *args, **kwargs): return else: # We don't actually delete records; we set them to inactive and then tell the translator to remove them - logging.info(f"Deactivating {self.route}") + logging.info("Deactivating", self.route) self.is_active = False self.save() diff --git a/translator/gobgp.py b/translator/gobgp.py index 5cd17ac4..cac40199 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -102,7 +102,7 @@ def _build_path(self, ip, event_data=None): comm_id = (asn << 16) + community communities.Pack(attribute_pb2.CommunitiesAttribute(communities=[comm_id])) else: - logging.info(f"LargeCommunity Used - ASN:{asn} Community: {community}") + logging.info("LargeCommunity Used - ASN:", asn, "Community:", community) global_admin = asn local_data1 = community # set to 0 because there's no use case for it, but we need a local_data2 for gobgp to read any of it @@ -124,7 +124,7 @@ def _build_path(self, ip, event_data=None): def add_path(self, ip, event_data): """Announce a single route.""" - logging.info(f"Blocking {ip}") + logging.info("Blocking", ip) try: path = self._build_path(ip, event_data) @@ -133,7 +133,7 @@ def add_path(self, ip, event_data): _TIMEOUT_SECONDS, ) except ASNError as e: - logging.warning(f"ASN assertion failed with error: {e}") + logging.warning("ASN assertion failed with error:", e) def del_all_paths(self): """Remove all routes from being announced.""" @@ -143,7 +143,7 @@ def del_all_paths(self): def del_path(self, ip, event_data): """Remove a single route from being announced.""" - logging.info(f"Unblocking {ip}") + logging.info("Unblocking", ip) try: path = self._build_path(ip, event_data) self.stub.DeletePath( @@ -151,7 +151,7 @@ def del_path(self, ip, event_data): _TIMEOUT_SECONDS, ) except ASNError as e: - logging.warning(f"ASN assertion failed with error: {e}") + logging.warning("ASN assertion failed with error:", e) def get_prefixes(self, ip): """Retrieve the routes that match a prefix and are announced.""" diff --git a/translator/translator.py b/translator/translator.py index abc8eab2..f787d590 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -32,7 +32,7 @@ def install_deps(): logging.info("Done installing dependencies for debuggers") - logging.info(f"Translator is set to use a debugger. Provided debug mode: {debug_mode}") + logging.info("Translator is set to use a debugger. Provided debug mode:", debug_mode) # We have to setup the debugger appropriately for various IDEs. It'd be nice if they all used the same thing but # sadly, we live in a fallen world. if debug_mode == "pycharm-pydevd": @@ -56,7 +56,7 @@ def install_deps(): logging.info("Debugger listening on port 56781.") else: - logging.warning(f"Invalid debug mode given: {debug_mode}. Debugger not started") + logging.warning("Invalid debug mode given:", debug_mode, "Debugger not started") # Must match the URL in asgi.py, and needs a trailing slash hostname = os.environ.get("SCRAM_HOSTNAME", "scram_hostname_not_set") @@ -78,7 +78,7 @@ async def main(): "translator_remove_all", "translator_check", ]: - logging.error(f"Unknown event type received: {event_type!r}") + logging.error("Unknown event type received:", event_type) # TODO: Maybe only allow this in testing? elif event_type == "translator_remove_all": g.del_all_paths() @@ -86,7 +86,7 @@ async def main(): try: ip = ipaddress.ip_interface(event_data["route"]) except: # noqa E722 - logging.error(f"Error parsing message: {message!r}") + logging.error("Error parsing message:", message) continue if event_type == "translator_add": From 37e93aebbcb60168849ef5995d413e11bbb381c8 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:37:27 -0600 Subject: [PATCH 107/156] Add PIE checks --- pyproject.toml | 1 + scram/route_manager/api/serializers.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 50fea91c..fb14b96c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ select = [ "ISC", # implicit-str-concat "LOG", # logging "N", # pep8-naming + "PIE", # pie "S", # bandit "SIM", # simplify "UP", # pyupgrade diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index 455f8f30..4c1294ee 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -17,7 +17,6 @@ class CustomCidrAddressField(rest_framework.CidrAddressField): """Define a wrapper field so swagger can properly handle the inherited field.""" - pass class ActionTypeSerializer(serializers.ModelSerializer): From f778f3a11772ccfd7f60ecf885aed4bf14d14399 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:41:57 -0600 Subject: [PATCH 108/156] Add raise checks --- pyproject.toml | 3 +++ scram/route_manager/api/serializers.py | 1 - scram/route_manager/api/views.py | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fb14b96c..cd01a69c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,8 +57,11 @@ select = [ "LOG", # logging "N", # pep8-naming "PIE", # pie + "Q", # quotes + "RSE", # raise "S", # bandit "SIM", # simplify + "T20", # print "UP", # pyupgrade ] ignore = [ diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index 4c1294ee..0618bd67 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -18,7 +18,6 @@ class CustomCidrAddressField(rest_framework.CidrAddressField): """Define a wrapper field so swagger can properly handle the inherited field.""" - class ActionTypeSerializer(serializers.ModelSerializer): """Map the serializer to the model via Meta.""" diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 98f47273..9f3129e3 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -107,7 +107,7 @@ def perform_create(self, serializer): # Make sure we put in an acceptable sized prefix min_prefix = getattr(settings, f"V{route.version}_MINPREFIX", 0) if route.prefixlen < min_prefix: - raise PrefixTooLarge() + raise PrefixTooLarge # Make sure this client is authorized to add this entry with this actiontype if self.request.data.get("uuid"): @@ -120,9 +120,9 @@ def perform_create(self, serializer): if not authorized_client or actiontype not in authorized_actiontypes: logging.debug("Client", client_uuid, "actiontypes:", authorized_actiontypes) logging.info(client_uuid, "is not allowed to add an entry to the", actiontype, "list") - raise ActiontypeNotAllowed() + raise ActiontypeNotAllowed elif not self.request.user.has_perm("route_manager.can_add_entry"): - raise PermissionDenied() + raise PermissionDenied # Don't process if we have the entry in the ignorelist overlapping_ignore = IgnoreEntry.objects.filter(route__net_overlaps=route) @@ -173,7 +173,7 @@ def find_entries(arg, active_filter=None): min_prefix = getattr(settings, f"V{cidr.version}_MINPREFIX", 0) if cidr.prefixlen < min_prefix: - raise PrefixTooLarge() from exc + raise PrefixTooLarge from exc query = Q(route__route__net_overlaps=cidr) From 5ed431b57cb24709e674097c658d28efb1963e0c Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:44:08 -0600 Subject: [PATCH 109/156] Add return checks --- pyproject.toml | 1 + scram/route_manager/api/views.py | 47 ++++++++++++++++---------------- scram/route_manager/models.py | 27 +++++++++--------- scram/route_manager/views.py | 2 +- translator/gobgp.py | 3 +- 5 files changed, 39 insertions(+), 41 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd01a69c..c1e80858 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ select = [ "PIE", # pie "Q", # quotes "RSE", # raise + "RET", # return "S", # bandit "SIM", # simplify "T20", # print diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 9f3129e3..df5f8518 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -132,30 +132,29 @@ def perform_create(self, serializer): ignore_entries.append(str(ignore_entry["route"])) logging.info("Cannot proceed adding", route, " The ignore list contains", ignore_entries) raise IgnoredRoute - else: - elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") - if not elements: - logging.warning("No elements found for actiontype:", actiontype) - - for element in elements: - msg = element.websocketmessage - msg.msg_data[msg.msg_data_route_field] = str(route) - # Must match a channel name defined in asgi.py - async_to_sync(channel_layer.group_send)( - f"translator_{actiontype}", - {"type": msg.msg_type, "message": msg.msg_data}, - ) - - serializer.save() - - entry = Entry.objects.get(route__route=route, actiontype__name=actiontype) - if expiration: - entry.expiration = expiration - entry.who = who - entry.is_active = True - entry.comment = comment - logging.info("Created entry:", entry) - entry.save() + elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") + if not elements: + logging.warning("No elements found for actiontype:", actiontype) + + for element in elements: + msg = element.websocketmessage + msg.msg_data[msg.msg_data_route_field] = str(route) + # Must match a channel name defined in asgi.py + async_to_sync(channel_layer.group_send)( + f"translator_{actiontype}", + {"type": msg.msg_type, "message": msg.msg_data}, + ) + + serializer.save() + + entry = Entry.objects.get(route__route=route, actiontype__name=actiontype) + if expiration: + entry.expiration = expiration + entry.who = who + entry.is_active = True + entry.comment = comment + logging.info("Created entry:", entry) + entry.save() @staticmethod def find_entries(arg, active_filter=None): diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index 35399820..914a93b9 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -120,20 +120,19 @@ def delete(self, *args, **kwargs): if not self.is_active: # We've already expired this route, don't send another message return - else: - # We don't actually delete records; we set them to inactive and then tell the translator to remove them - logging.info("Deactivating", self.route) - self.is_active = False - self.save() - - # Unblock it - async_to_sync(channel_layer.group_send)( - f"translator_{self.actiontype}", - { - "type": "translator_remove", - "message": {"route": str(self.route)}, - }, - ) + # We don't actually delete records; we set them to inactive and then tell the translator to remove them + logging.info("Deactivating", self.route) + self.is_active = False + self.save() + + # Unblock it + async_to_sync(channel_layer.group_send)( + f"translator_{self.actiontype}", + { + "type": "translator_remove", + "message": {"route": str(self.route)}, + }, + ) def get_change_reason(self): """Traverse come complex relationships to determine the most recent change reason.""" diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 827aabba..244ac37b 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -127,7 +127,7 @@ def add_entry(request): messages.add_message(request, messages.WARNING, f"Something went wrong: {res.status_code}") with transaction.atomic(): home = home_page(request) - return home + return home # noqa RET504 def process_expired(request): diff --git a/translator/gobgp.py b/translator/gobgp.py index cac40199..3e1215c7 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -30,8 +30,7 @@ def __init__(self, url): def _get_family_afi(self, ip_version): if ip_version == 6: return gobgp_pb2.Family.AFI_IP6 - else: - return gobgp_pb2.Family.AFI_IP + return gobgp_pb2.Family.AFI_IP def _build_path(self, ip, event_data=None): # Grab ASN and Community from our event_data, or use the defaults From b195a9b703d13ad8988739fc14e0710f0408b809 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:45:30 -0600 Subject: [PATCH 110/156] Add self checks --- pyproject.toml | 1 + scram/route_manager/tests/test_history.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1e80858..9079de3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ select = [ "RET", # return "S", # bandit "SIM", # simplify + "SLF", # self "T20", # print "UP", # pyupgrade ] diff --git a/scram/route_manager/tests/test_history.py b/scram/route_manager/tests/test_history.py index b234e621..d2848e83 100644 --- a/scram/route_manager/tests/test_history.py +++ b/scram/route_manager/tests/test_history.py @@ -16,7 +16,7 @@ def setUp(self): def test_comments(self): """Ensure we can go back and set a reason.""" self.atype.name = "Nullroute" - self.atype._change_reason = "Use more descriptive name" + self.atype._change_reason = "Use more descriptive name" # noqa SLF001 self.atype.save() self.assertIsNotNone(get_change_reason_from_object(self.atype)) @@ -47,7 +47,7 @@ def test_comments(self): e.route = Route.objects.create(route=route_new) change_reason = "I meant 32, not 16." - e._change_reason = change_reason + e._change_reason = change_reason # noqa SLF001 e.save() self.assertEqual(len(e.history.all()), 2) From 5b23826272076708da788be7b5546e07a42a63a1 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 12:48:58 -0600 Subject: [PATCH 111/156] Add eradicate checks --- config/wsgi.py | 5 +---- pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/wsgi.py b/config/wsgi.py index b679ee04..73ed363b 100644 --- a/config/wsgi.py +++ b/config/wsgi.py @@ -26,13 +26,10 @@ # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks # if running multiple sites in the same mod_wsgi process. To fix this, use # mod_wsgi daemon mode with each site in its own daemon process, or use -# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production" +# os.environ["DJANGO_SETTINGS_MODULE"] = "config.settings.production" # noqa ERA001 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.production") # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. application = get_wsgi_application() -# Apply WSGI middleware here. -# from helloworld.wsgi import HelloWorldApplication -# application = HelloWorldApplication(application) diff --git a/pyproject.toml b/pyproject.toml index 9079de3c..1770c6d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ select = [ "DTZ", # datetimez "E", # pycodestyle "EM", # errmsg + "ERA", # eradicate "F", # pyflakes "FBT", # boolean-trap "G", # logging-format @@ -57,12 +58,14 @@ select = [ "LOG", # logging "N", # pep8-naming "PIE", # pie + "PTH", # use-pathlib "Q", # quotes "RSE", # raise "RET", # return "S", # bandit "SIM", # simplify "SLF", # self + "SLOT", # slots "T20", # print "UP", # pyupgrade ] From 2c0b461df634f5994f7cf13d2ab61c71c1a8622a Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 13:10:01 -0600 Subject: [PATCH 112/156] Add pylint checks --- config/asgi.py | 4 ++-- config/consumers.py | 2 +- config/settings/base.py | 2 +- pyproject.toml | 4 +++- scram/route_manager/api/serializers.py | 2 +- scram/route_manager/api/views.py | 12 ++++++------ scram/route_manager/models.py | 2 +- .../tests/acceptance/steps/common.py | 2 +- .../tests/test_autocreate_admin.py | 15 +++++++++------ scram/route_manager/tests/test_swagger.py | 6 +++--- scram/route_manager/views.py | 6 +++--- translator/gobgp.py | 19 +++++++++++-------- translator/shared.py | 4 +++- translator/translator.py | 8 ++++---- 14 files changed, 49 insertions(+), 39 deletions(-) diff --git a/config/asgi.py b/config/asgi.py index 6e6b8979..a38b268e 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -21,7 +21,7 @@ # Here we setup a debugger if this is desired. This obviously should not be run in production. debug = os.environ.get("DEBUG") if debug: - logging.info("Django is set to use a debugger. Provided debug mode:", debug) + logging.info("Django is set to use a debugger. Provided debug mode: %s", debug) if debug == "pycharm-pydevd": logging.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") @@ -39,7 +39,7 @@ logging.info("Debugger listening on port 56780.") else: - logging.warning("Invalid debug mode given: %s.", "Debugger not started", extra=debug) + logging.warning("Invalid debug mode given: %s. Debugger not started", debug) # This allows easy placement of apps within the interior # scram directory. diff --git a/config/consumers.py b/config/consumers.py index 082b31b2..1787707b 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -40,7 +40,7 @@ async def connect(self): async def disconnect(self, close_code): """Discard any remaining messages on disconnect.""" - logging.info("Disconnect received:", close_code) + logging.info("Disconnect received: %s", close_code) await self.channel_layer.group_discard(self.translator_group, self.channel_name) async def receive_json(self, content): diff --git a/config/settings/base.py b/config/settings/base.py index 1861af43..21b3fe1e 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -308,7 +308,7 @@ ) OIDC_RP_SIGN_ALGO = "RS256" -logging.info("Using AUTH METHOD =", AUTH_METHOD) +logging.info("Using AUTH METHOD=%s", AUTH_METHOD) if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 diff --git a/pyproject.toml b/pyproject.toml index 1770c6d3..0b2128f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ select = [ "LOG", # logging "N", # pep8-naming "PIE", # pie + "PL", # pylint "PTH", # use-pathlib "Q", # quotes "RSE", # raise @@ -82,8 +83,9 @@ ignore = [ "test.py" = [ "S105", # hardcoded password as argument ] -"factories.py" = [ +"scram/users/**" = [ "FBT001", # minimal issue; don't need to mess with in the User app + "PLR2004", # magic values when checking HTTP status codes ] # ==== isort ==== diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index 0618bd67..a4f6d525 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -89,7 +89,7 @@ def create(self, validated_data): actiontype_instance = ActionType.objects.get(name=actiontype) entry_instance, created = Entry.objects.get_or_create(route=route_instance, actiontype=actiontype_instance) - logger.debug("Created entry with comment:", comment) + logger.debug("Created entry with comment: %s", comment) update_change_reason(entry_instance, comment) return entry_instance diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index df5f8518..d4b819c6 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -102,7 +102,7 @@ def perform_create(self, serializer): try: expiration = parse_datetime(tmp_exp) # noqa: F841 except ValueError: - logging.warning("Could not parse expiration DateTime string:", tmp_exp) + logging.warning("Could not parse expiration DateTime string: %s", tmp_exp) # Make sure we put in an acceptable sized prefix min_prefix = getattr(settings, f"V{route.version}_MINPREFIX", 0) @@ -118,8 +118,8 @@ def perform_create(self, serializer): ) authorized_client = Client.objects.filter(uuid=client_uuid).values("is_authorized") if not authorized_client or actiontype not in authorized_actiontypes: - logging.debug("Client", client_uuid, "actiontypes:", authorized_actiontypes) - logging.info(client_uuid, "is not allowed to add an entry to the", actiontype, "list") + logging.debug("Client: %s, actiontypes: %s", client_uuid, authorized_actiontypes) + logging.info("%s is not allowed to add an entry to the %s list.", client_uuid, actiontype) raise ActiontypeNotAllowed elif not self.request.user.has_perm("route_manager.can_add_entry"): raise PermissionDenied @@ -130,11 +130,11 @@ def perform_create(self, serializer): ignore_entries = [] for ignore_entry in overlapping_ignore.values(): ignore_entries.append(str(ignore_entry["route"])) - logging.info("Cannot proceed adding", route, " The ignore list contains", ignore_entries) + logging.info("Cannot proceed adding %s. The ignore list contains %s.", route, ignore_entries) raise IgnoredRoute elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") if not elements: - logging.warning("No elements found for actiontype:", actiontype) + logging.warning("No elements found for actiontype: %s", actiontype) for element in elements: msg = element.websocketmessage @@ -153,7 +153,7 @@ def perform_create(self, serializer): entry.who = who entry.is_active = True entry.comment = comment - logging.info("Created entry:", entry) + logging.info("Created entry: %s", entry) entry.save() @staticmethod diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index 914a93b9..9c123cca 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -121,7 +121,7 @@ def delete(self, *args, **kwargs): # We've already expired this route, don't send another message return # We don't actually delete records; we set them to inactive and then tell the translator to remove them - logging.info("Deactivating", self.route) + logging.info("Deactivating %s", self.route) self.is_active = False self.save() diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index d4836fe9..a81e5a33 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -3,10 +3,10 @@ import datetime import time -import django.conf as conf from asgiref.sync import async_to_sync from behave import given, step, then, when from channels.layers import get_channel_layer +from django import conf from django.urls import reverse from scram.route_manager.models import ActionType, Client, WebSocketMessage, WebSocketSequenceElement diff --git a/scram/route_manager/tests/test_autocreate_admin.py b/scram/route_manager/tests/test_autocreate_admin.py index 7a83094f..88270f4e 100644 --- a/scram/route_manager/tests/test_autocreate_admin.py +++ b/scram/route_manager/tests/test_autocreate_admin.py @@ -7,6 +7,9 @@ from scram.users.models import User +LEVEL_SUCCESS = 25 +LEVEL_INFO = 20 + @pytest.mark.django_db def test_autocreate_admin(settings): @@ -14,15 +17,15 @@ def test_autocreate_admin(settings): settings.AUTOCREATE_ADMIN = True client = Client() response = client.get(reverse("route_manager:home")) - assert response.status_code == 200 + assert response.status_code == 200 # noqa: PLR2004 assert User.objects.count() == 1 user = User.objects.get(username="admin") assert user.is_superuser assert user.email == "admin@example.com" messages = list(get_messages(response.wsgi_request)) - assert len(messages) == 2 - assert messages[0].level == 25 # SUCCESS - assert messages[1].level == 20 # INFO + assert len(messages) == 2 # noqa: PLR2004 + assert messages[0].level == LEVEL_SUCCESS + assert messages[1].level == LEVEL_INFO @pytest.mark.django_db @@ -31,7 +34,7 @@ def test_autocreate_admin_disabled(settings): settings.AUTOCREATE_ADMIN = False client = Client() response = client.get(reverse("route_manager:home")) - assert response.status_code == 200 + assert response.status_code == 200 # noqa: PLR2004 assert User.objects.count() == 0 @@ -42,6 +45,6 @@ def test_autocreate_admin_existing_user(settings): User.objects.create_user("testuser", "test@example.com", "password") client = Client() response = client.get(reverse("route_manager:home")) - assert response.status_code == 200 + assert response.status_code == 200 # noqa: PLR2004 assert User.objects.count() == 1 assert not User.objects.filter(username="admin").exists() diff --git a/scram/route_manager/tests/test_swagger.py b/scram/route_manager/tests/test_swagger.py index aae60e9b..ce955db9 100644 --- a/scram/route_manager/tests/test_swagger.py +++ b/scram/route_manager/tests/test_swagger.py @@ -9,7 +9,7 @@ def test_swagger_api(client): """Test that the Swagger API endpoint returns a successful response.""" url = reverse("swagger-ui") response = client.get(url) - assert response.status_code == 200 + assert response.status_code == 200 # noqa: PLR2004 @pytest.mark.django_db @@ -17,7 +17,7 @@ def test_redoc_api(client): """Test that the Redoc API endpoint returns a successful response.""" url = reverse("redoc") response = client.get(url) - assert response.status_code == 200 + assert response.status_code == 200 # noqa: PLR2004 @pytest.mark.django_db @@ -25,6 +25,6 @@ def test_schema_api(client): """Test that the Schema API endpoint returns a successful response.""" url = reverse("schema") response = client.get(url) - assert response.status_code == 200 + assert response.status_code == 200 # noqa: PLR2004 expected_strings = [b"/entries/", b"/actiontypes/", b"/ignore_entries/", b"/users/"] assert all(string in response.content for string in expected_strings) diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 244ac37b..1ff894a4 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -105,13 +105,13 @@ def add_entry(request): with transaction.atomic(): res = add_entry_api(request) - if res.status_code == 201: + if res.status_code == 201: # noqa: PLR2004 messages.add_message( request, messages.SUCCESS, "Entry successfully added", ) - elif res.status_code == 400: + elif res.status_code == 400: # noqa: PLR2004 errors = [] if isinstance(res.data, rest_framework.utils.serializer_helpers.ReturnDict): for k, v in res.data.items(): @@ -121,7 +121,7 @@ def add_entry(request): for v in res.data.values(): errors.append(f"error: {str(v)}") messages.add_message(request, messages.ERROR, "
".join(errors)) - elif res.status_code == 403: + elif res.status_code == 403: # noqa: PLR2004 messages.add_message(request, messages.ERROR, "Permission Denied") else: messages.add_message(request, messages.WARNING, f"Something went wrong: {res.status_code}") diff --git a/translator/gobgp.py b/translator/gobgp.py index 3e1215c7..e67a92d1 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -15,6 +15,9 @@ DEFAULT_COMMUNITY = 666 DEFAULT_V4_NEXTHOP = "192.0.2.199" DEFAULT_V6_NEXTHOP = "100::1" +MAX_SMALL_ASN = 2**16 +MAX_SMALL_COMM = 2**16 +IPV6 = 6 logging.basicConfig(level=logging.DEBUG) @@ -28,7 +31,7 @@ def __init__(self, url): self.stub = gobgp_pb2_grpc.GobgpApiStub(channel) def _get_family_afi(self, ip_version): - if ip_version == 6: + if ip_version == IPV6: return gobgp_pb2.Family.AFI_IP6 return gobgp_pb2.Family.AFI_IP @@ -63,7 +66,7 @@ def _build_path(self, ip, event_data=None): # Set the next hop to the correct value depending on IP family next_hop = Any() family_afi = self._get_family_afi(ip_version) - if ip_version == 6: + if ip_version == IPV6: next_hops = event_data.get("next_hop", DEFAULT_V6_NEXTHOP) next_hop.Pack( attribute_pb2.MpReachNLRIAttribute( @@ -95,13 +98,13 @@ def _build_path(self, ip, event_data=None): communities = Any() # Standard community # Since we pack both into the community string we need to make sure they will both fit - if asn < 65536 and community < 65536: + if asn < MAX_SMALL_ASN and community < MAX_SMALL_COMM: # We bitshift ASN left by 16 so that there is room to add the community on the end of it. This is because # GoBGP wants the community sent as a single integer. comm_id = (asn << 16) + community communities.Pack(attribute_pb2.CommunitiesAttribute(communities=[comm_id])) else: - logging.info("LargeCommunity Used - ASN:", asn, "Community:", community) + logging.info("LargeCommunity Used - ASN: %s. Community: %s", asn, community) global_admin = asn local_data1 = community # set to 0 because there's no use case for it, but we need a local_data2 for gobgp to read any of it @@ -123,7 +126,7 @@ def _build_path(self, ip, event_data=None): def add_path(self, ip, event_data): """Announce a single route.""" - logging.info("Blocking", ip) + logging.info("Blocking %s", ip) try: path = self._build_path(ip, event_data) @@ -132,7 +135,7 @@ def add_path(self, ip, event_data): _TIMEOUT_SECONDS, ) except ASNError as e: - logging.warning("ASN assertion failed with error:", e) + logging.warning("ASN assertion failed with error: %s", e) def del_all_paths(self): """Remove all routes from being announced.""" @@ -142,7 +145,7 @@ def del_all_paths(self): def del_path(self, ip, event_data): """Remove a single route from being announced.""" - logging.info("Unblocking", ip) + logging.info("Unblocking %s", ip) try: path = self._build_path(ip, event_data) self.stub.DeletePath( @@ -150,7 +153,7 @@ def del_path(self, ip, event_data): _TIMEOUT_SECONDS, ) except ASNError as e: - logging.warning("ASN assertion failed with error:", e) + logging.warning("ASN assertion failed with error: %s", e) def get_prefixes(self, ip): """Retrieve the routes that match a prefix and are announced.""" diff --git a/translator/shared.py b/translator/shared.py index ebe56e6d..78555b2d 100644 --- a/translator/shared.py +++ b/translator/shared.py @@ -2,6 +2,8 @@ from exceptions import ASNError +MAX_ASN_VAL = 2**32 - 1 + def asn_is_valid(asn: int) -> bool: """asn_is_valid makes sure that an ASN passed in is a valid 2 or 4 Byte ASN. @@ -19,7 +21,7 @@ def asn_is_valid(asn: int) -> bool: if not isinstance(asn, int): msg = f"ASN {asn} is not an Integer, has type {type(asn)}" raise ASNError(msg) - if not 0 < asn < 4294967295: + if not 0 < asn < MAX_ASN_VAL: # This is the max as stated in rfc6996 msg = f"ASN {asn} is out of range. Must be between 0 and 4294967295" raise ASNError(msg) diff --git a/translator/translator.py b/translator/translator.py index f787d590..17c3651d 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -32,7 +32,7 @@ def install_deps(): logging.info("Done installing dependencies for debuggers") - logging.info("Translator is set to use a debugger. Provided debug mode:", debug_mode) + logging.info("Translator is set to use a debugger. Provided debug mode: %s", debug_mode) # We have to setup the debugger appropriately for various IDEs. It'd be nice if they all used the same thing but # sadly, we live in a fallen world. if debug_mode == "pycharm-pydevd": @@ -56,7 +56,7 @@ def install_deps(): logging.info("Debugger listening on port 56781.") else: - logging.warning("Invalid debug mode given:", debug_mode, "Debugger not started") + logging.warning("Invalid debug mode given: %s. Debugger not started", debug_mode) # Must match the URL in asgi.py, and needs a trailing slash hostname = os.environ.get("SCRAM_HOSTNAME", "scram_hostname_not_set") @@ -78,7 +78,7 @@ async def main(): "translator_remove_all", "translator_check", ]: - logging.error("Unknown event type received:", event_type) + logging.error("Unknown event type received: %s", event_type) # TODO: Maybe only allow this in testing? elif event_type == "translator_remove_all": g.del_all_paths() @@ -86,7 +86,7 @@ async def main(): try: ip = ipaddress.ip_interface(event_data["route"]) except: # noqa E722 - logging.error("Error parsing message:", message) + logging.error("Error parsing message: %s", message) continue if event_type == "translator_add": From 0bfc94537c5b467e5614a6d77e6fc07c78294bdc Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 13:10:58 -0600 Subject: [PATCH 113/156] Add tryceratops checks --- pyproject.toml | 1 + translator/translator.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0b2128f7..5317d330 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ select = [ "SLF", # self "SLOT", # slots "T20", # print + "TRY", # tryceratops "UP", # pyupgrade ] ignore = [ diff --git a/translator/translator.py b/translator/translator.py index 17c3651d..5c9f22f4 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -86,7 +86,7 @@ async def main(): try: ip = ipaddress.ip_interface(event_data["route"]) except: # noqa E722 - logging.error("Error parsing message: %s", message) + logging.exception("Error parsing message: %s", message) continue if event_type == "translator_add": From a2d6e6c2cd8bf6e097678a66552c1f3492474004 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 13:17:53 -0600 Subject: [PATCH 114/156] Add perflint checks --- pyproject.toml | 2 ++ scram/route_manager/api/views.py | 4 +--- scram/route_manager/views.py | 6 ++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5317d330..3a4c2a2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,12 +51,14 @@ select = [ "ERA", # eradicate "F", # pyflakes "FBT", # boolean-trap + "FLY", # flynt "G", # logging-format "I", # isort "ICN", # import-conventions "ISC", # implicit-str-concat "LOG", # logging "N", # pep8-naming + "PERF", # perflint "PIE", # pie "PL", # pylint "PTH", # use-pathlib diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index d4b819c6..47520ada 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -127,9 +127,7 @@ def perform_create(self, serializer): # Don't process if we have the entry in the ignorelist overlapping_ignore = IgnoreEntry.objects.filter(route__net_overlaps=route) if overlapping_ignore.count(): - ignore_entries = [] - for ignore_entry in overlapping_ignore.values(): - ignore_entries.append(str(ignore_entry["route"])) + ignore_entries = [str(ignore_entry["route"]) for ignore_entry in overlapping_ignore.values()] logging.info("Cannot proceed adding %s. The ignore list contains %s.", route, ignore_entries) raise IgnoredRoute elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 1ff894a4..c1388422 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -115,11 +115,9 @@ def add_entry(request): errors = [] if isinstance(res.data, rest_framework.utils.serializer_helpers.ReturnDict): for k, v in res.data.items(): - for error in v: - errors.append(f"'{k}' error: {str(error)}") + errors.extend(f"'{k}' error: {str(error)}" for error in v) else: - for v in res.data.values(): - errors.append(f"error: {str(v)}") + errors.extend(f"error: {str(v)}" for v in res.data.values()) messages.add_message(request, messages.ERROR, "
".join(errors)) elif res.status_code == 403: # noqa: PLR2004 messages.add_message(request, messages.ERROR, "Permission Denied") From 332e002df04d7cd2c9c2afed1af91fdfd17ece2c Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 13:25:09 -0600 Subject: [PATCH 115/156] Add ruff checks --- config/settings/base.py | 6 +++--- config/settings/local.py | 6 +++--- config/urls.py | 5 +++-- pyproject.toml | 5 ++++- scram/route_manager/api/views.py | 2 +- scram/route_manager/views.py | 4 ++-- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 21b3fe1e..65ad2c95 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -263,7 +263,7 @@ # ------------------------------------------------------------------------------- # django-rest-framework - https://www.django-rest-framework.org/api-guide/settings/ # Swagger related tooling -INSTALLED_APPS += ["drf_spectacular"] # noqa F405 +INSTALLED_APPS += ["drf_spectacular"] REST_FRAMEWORK = { "DEFAULT_AUTHENTICATION_CLASSES": ( "rest_framework.authentication.SessionAuthentication", @@ -311,10 +311,10 @@ logging.info("Using AUTH METHOD=%s", AUTH_METHOD) if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware - MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # noqa F405 + MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] # Extend middleware to add OIDC auth backend - AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # noqa F405 + AUTHENTICATION_BACKENDS += ["scram.route_manager.authentication_backends.ESnetAuthBackend"] # https://docs.djangoproject.com/en/dev/ref/settings/#login-url LOGIN_URL = "oidc_authentication_init" diff --git a/config/settings/local.py b/config/settings/local.py index 7b8ef9fc..de20d800 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -41,7 +41,7 @@ # django-debug-toolbar # ------------------------------------------------------------------------------ # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#prerequisites -INSTALLED_APPS += ["debug_toolbar"] # noqa F405 +INSTALLED_APPS += ["debug_toolbar"] # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#middleware MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"] # noqa F405 # https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config @@ -60,14 +60,14 @@ # django-extensions # ------------------------------------------------------------------------------ # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration -INSTALLED_APPS += ["django_extensions"] # noqa F405 +INSTALLED_APPS += ["django_extensions"] # Your stuff... # ------------------------------------------------------------------------------ REST_FRAMEWORK["DEFAULT_PERMISSION_CLASSES"] = ("rest_framework.permissions.IsAdminUser",) # Behave Django testing framework -INSTALLED_APPS += ["behave_django"] # noqa F405 +INSTALLED_APPS += ["behave_django"] # AUTHENTICATION # ------------------------------------------------------------------------------ diff --git a/config/urls.py b/config/urls.py index a10e0dbd..41a45d12 100644 --- a/config/urls.py +++ b/config/urls.py @@ -19,7 +19,8 @@ # User management path("users/", include("scram.users.urls", namespace="users")), # Your stuff: custom urls includes go here -] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) + *static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT), +] if settings.DEBUG: # Static file serving when using Gunicorn + Uvicorn for local web socket development urlpatterns += staticfiles_urlpatterns() @@ -78,4 +79,4 @@ if "debug_toolbar" in settings.INSTALLED_APPS: import debug_toolbar - urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns + urlpatterns = [path("__debug__/", include(debug_toolbar.urls)), *urlpatterns] diff --git a/pyproject.toml b/pyproject.toml index 3a4c2a2d..677ed3a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ select = [ "COM", # commas "D", # pydocstyle "DJ", # django + "DOC", # pydoclint "DTZ", # datetimez "E", # pycodestyle "EM", # errmsg @@ -63,8 +64,9 @@ select = [ "PL", # pylint "PTH", # use-pathlib "Q", # quotes - "RSE", # raise "RET", # return + "RSE", # raise + "RUF", # Ruff "S", # bandit "SIM", # simplify "SLF", # self @@ -74,6 +76,7 @@ select = [ "UP", # pyupgrade ] ignore = [ + "RUF012", # need more widespread typing "SIM102", # Use a single `if` statement instead of nested `if` statements "SIM108", # Use ternary operator instead of `if`-`else`-block ] diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 47520ada..d868edf6 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -100,7 +100,7 @@ def perform_create(self, serializer): tmp_exp = self.request.data.get("expiration", "") try: - expiration = parse_datetime(tmp_exp) # noqa: F841 + expiration = parse_datetime(tmp_exp) except ValueError: logging.warning("Could not parse expiration DateTime string: %s", tmp_exp) diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index c1388422..80260506 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -115,9 +115,9 @@ def add_entry(request): errors = [] if isinstance(res.data, rest_framework.utils.serializer_helpers.ReturnDict): for k, v in res.data.items(): - errors.extend(f"'{k}' error: {str(error)}" for error in v) + errors.extend(f"'{k}' error: {error!s}" for error in v) else: - errors.extend(f"error: {str(v)}" for v in res.data.values()) + errors.extend(f"error: {v!s}" for v in res.data.values()) messages.add_message(request, messages.ERROR, "
".join(errors)) elif res.status_code == 403: # noqa: PLR2004 messages.add_message(request, messages.ERROR, "Permission Denied") From 596d095c475e9f82bccec6570ed785c4eeb5aa9e Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 13:27:01 -0600 Subject: [PATCH 116/156] Final cleanup --- scram/route_manager/tests/functional_tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scram/route_manager/tests/functional_tests.py b/scram/route_manager/tests/functional_tests.py index c4515b13..a87ce21e 100644 --- a/scram/route_manager/tests/functional_tests.py +++ b/scram/route_manager/tests/functional_tests.py @@ -6,8 +6,6 @@ class HomePageTest(unittest.TestCase): """Ensure the home page works.""" - pass - if __name__ == "__main__": unittest.main(warnings="ignore") From af2f7ed33f86ec5f18408fd3d88a6f7453f889f9 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 14:01:52 -0600 Subject: [PATCH 117/156] Add complexity checks --- pyproject.toml | 4 ++ scram/route_manager/api/views.py | 46 +++++++------ .../route_manager/authentication_backends.py | 33 +++------- translator/translator.py | 64 +++++++++++-------- 4 files changed, 77 insertions(+), 70 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 677ed3a8..5bc5458f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ select = [ "B", # bugbear "BLE", # blind-except "C4", # comprehensions + "C90", # complexity "COM", # commas "D", # pydocstyle "DJ", # django @@ -81,6 +82,9 @@ ignore = [ "SIM108", # Use ternary operator instead of `if`-`else`-block ] +[tool.ruff.lint.mccabe] +max-complexity = 7 # our current code adheres to this without too much effort + [tool.ruff.lint.per-file-ignores] "**/{tests}/*" = [ "S101", # use of assert diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index d868edf6..802e82d8 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -86,6 +86,30 @@ def get_permissions(self): return [AllowAny()] return super().get_permissions() + def check_client_authorization(self, actiontype): + """Ensure that a given client is authorized to use a given actiontype.""" + uuid = self.request.data.get("uuid") + if uuid: + authorized_actiontypes = Client.objects.filter(uuid=uuid).values_list( + "authorized_actiontypes__name", + flat=True, + ) + authorized_client = Client.objects.filter(uuid=uuid).values("is_authorized") + if not authorized_client or actiontype not in authorized_actiontypes: + logging.debug("Client: %s, actiontypes: %s", uuid, authorized_actiontypes) + logging.info("%s is not allowed to add an entry to the %s list.", uuid, actiontype) + raise ActiontypeNotAllowed + elif not self.request.user.has_perm("route_manager.can_add_entry"): + raise PermissionDenied + + def check_ignore_list(self, route): + """Ensure that we're not trying to block something from the ignore list.""" + overlapping_ignore = IgnoreEntry.objects.filter(route__net_overlaps=route) + if overlapping_ignore.count(): + ignore_entries = [str(ignore_entry["route"]) for ignore_entry in overlapping_ignore.values()] + logging.info("Cannot proceed adding %s. The ignore list contains %s.", route, ignore_entries) + raise IgnoredRoute + def perform_create(self, serializer): """Create a new Entry, causing that route to receive the actiontype (i.e. block).""" actiontype = serializer.validated_data["actiontype"] @@ -109,27 +133,9 @@ def perform_create(self, serializer): if route.prefixlen < min_prefix: raise PrefixTooLarge - # Make sure this client is authorized to add this entry with this actiontype - if self.request.data.get("uuid"): - client_uuid = self.request.data["uuid"] - authorized_actiontypes = Client.objects.filter(uuid=client_uuid).values_list( - "authorized_actiontypes__name", - flat=True, - ) - authorized_client = Client.objects.filter(uuid=client_uuid).values("is_authorized") - if not authorized_client or actiontype not in authorized_actiontypes: - logging.debug("Client: %s, actiontypes: %s", client_uuid, authorized_actiontypes) - logging.info("%s is not allowed to add an entry to the %s list.", client_uuid, actiontype) - raise ActiontypeNotAllowed - elif not self.request.user.has_perm("route_manager.can_add_entry"): - raise PermissionDenied + self.check_client_authorization(actiontype) + self.check_ignore_list(route) - # Don't process if we have the entry in the ignorelist - overlapping_ignore = IgnoreEntry.objects.filter(route__net_overlaps=route) - if overlapping_ignore.count(): - ignore_entries = [str(ignore_entry["route"]) for ignore_entry in overlapping_ignore.values()] - logging.info("Cannot proceed adding %s. The ignore list contains %s.", route, ignore_entries) - raise IgnoredRoute elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") if not elements: logging.warning("No elements found for actiontype: %s", actiontype) diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index 093bfec4..e7793455 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -10,31 +10,18 @@ class ESnetAuthBackend(OIDCAuthenticationBackend): def update_groups(self, user, claims): """Set the user's group(s) to whatever is in the claims.""" - claimed_groups = claims.get("groups", []) - effective_groups = [] - is_admin = False - - ro_group = Group.objects.get(name="readonly") - rw_group = Group.objects.get(name="readwrite") - - for g in claimed_groups: - # If any of the user's groups are in DENIED_GROUPS, deny them and stop processing immediately - if g in settings.SCRAM_DENIED_GROUPS: - effective_groups = [] - is_admin = False - break - - if g in settings.SCRAM_ADMIN_GROUPS: - is_admin = True - - if g in settings.SCRAM_READONLY_GROUPS: - if ro_group not in effective_groups: - effective_groups.append(ro_group) + claimed_groups = claims.get("groups", []) - if g in settings.SCRAM_READWRITE_GROUPS: - if rw_group not in effective_groups: - effective_groups.append(rw_group) + if any(claimed_groups in settings.SCRAM_DENIED_GROUPS): + is_admin = False + # Don't even look at anything else if they're denied + else: + is_admin = any(claimed_groups in settings.SCRAM_ADMIN_GROUPS) + if any(claimed_groups in settings.SCRAM_READWRITE_GROUPS): + effective_groups += Group.objects.get(name="readwrite") + if any(claimed_groups in settings.SCRAM_READONLY_GROUPS): + effective_groups += Group.objects.get(name="readonly") user.groups.set(effective_groups) user.is_staff = user.is_superuser = is_admin diff --git a/translator/translator.py b/translator/translator.py index 5c9f22f4..ef168c60 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -61,6 +61,25 @@ def install_deps(): # Must match the URL in asgi.py, and needs a trailing slash hostname = os.environ.get("SCRAM_HOSTNAME", "scram_hostname_not_set") url = os.environ.get("SCRAM_EVENTS_URL", "ws://django:8000/ws/route_manager/translator_block/") +KNOWN_MESSAGES = [ + "translator_add", + "translator_remove", + "translator_remove_all", + "translator_check", +] + + +async def block_or_unblock(gobgp, event_type, event_data, ip): + """Receive one message and pass it off to the right function.""" + # TODO: Maybe only allow this in testing? + if event_type == "translator_remove_all": + gobgp.del_all_paths() + return + + if event_type == "translator_add": + gobgp.add_path(ip, event_data) + elif event_type == "translator_remove": + gobgp.del_path(ip, event_data) async def main(): @@ -69,35 +88,26 @@ async def main(): async for websocket in websockets.connect(url): try: async for message in websocket: - json_message = json.loads(message) - event_type = json_message.get("type") - event_data = json_message.get("message") - if event_type not in [ - "translator_add", - "translator_remove", - "translator_remove_all", - "translator_check", - ]: + raw_message = json.loads(message) + event_type = raw_message.get("type") + event_data = raw_message.get("message") + if event_type not in KNOWN_MESSAGES: logging.error("Unknown event type received: %s", event_type) - # TODO: Maybe only allow this in testing? - elif event_type == "translator_remove_all": - g.del_all_paths() + continue + + try: + ip = ipaddress.ip_interface(event_data["route"]) + except: # noqa E722 + logging.exception("Error parsing message: %s", raw_message) + continue + + if event_type == "translator_check": + raw_message["type"] = "translator_check_resp" + raw_message["message"]["is_blocked"] = g.is_blocked(ip) + raw_message["message"]["translator_name"] = hostname + await websocket.send(json.dumps(raw_message)) else: - try: - ip = ipaddress.ip_interface(event_data["route"]) - except: # noqa E722 - logging.exception("Error parsing message: %s", message) - continue - - if event_type == "translator_add": - g.add_path(ip, event_data) - elif event_type == "translator_remove": - g.del_path(ip, event_data) - elif event_type == "translator_check": - json_message["type"] = "translator_check_resp" - json_message["message"]["is_blocked"] = g.is_blocked(ip) - json_message["message"]["translator_name"] = hostname - await websocket.send(json.dumps(json_message)) + await block_or_unblock(g, event_type, event_data, ip) except websockets.ConnectionClosed: continue From 9b9b72c06d239cc06bae095781b03c20989378ce Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 14:46:17 -0600 Subject: [PATCH 118/156] Add preview ruff checks --- .pre-commit-config.yaml | 22 ++---- config/asgi.py | 14 ++-- config/consumers.py | 8 +- config/settings/base.py | 4 +- docs/.#summary.md | 1 + docs/README.md | 2 +- manage.py | 2 +- pyproject.toml | 73 +++---------------- scram/__init__.py | 2 +- scram/route_manager/api/serializers.py | 18 +++-- scram/route_manager/api/views.py | 16 ++-- .../route_manager/authentication_backends.py | 7 +- scram/route_manager/context_processors.py | 6 +- scram/route_manager/models.py | 29 +++++--- .../tests/acceptance/steps/common.py | 8 +- .../tests/acceptance/steps/translator.py | 2 +- scram/route_manager/views.py | 3 +- scram/users/api/views.py | 3 +- scram/users/apps.py | 3 +- scram/users/tests/test_admin.py | 10 +-- scram/users/tests/test_drf_views.py | 15 ++-- scram/users/tests/test_forms.py | 8 +- scram/users/tests/test_views.py | 14 ++-- scram/utils/context_processors.py | 6 +- translator/gobgp.py | 24 +++--- translator/tests/acceptance/steps/actions.py | 6 +- translator/translator.py | 30 ++++---- 27 files changed, 165 insertions(+), 171 deletions(-) create mode 120000 docs/.#summary.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48d2a7b2..08d8da08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -exclude: 'docs|node_modules|migrations|.git|.tox' +exclude: 'docs|migrations|.git|.tox' default_stages: [commit] fail_fast: true @@ -9,17 +9,11 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - - repo: https://github.com/psf/black - rev: 23.9.1 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.0 hooks: - - id: black - - - repo: https://github.com/timothycrosley/isort - rev: 5.12.0 - hooks: - - id: isort - - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format diff --git a/config/asgi.py b/config/asgi.py index a38b268e..5e11d1fa 100644 --- a/config/asgi.py +++ b/config/asgi.py @@ -18,28 +18,30 @@ # TODO: from channels.security.websocket import AllowedHostsOriginValidator from django.core.asgi import get_asgi_application +logger = logging.getLogger(__name__) + # Here we setup a debugger if this is desired. This obviously should not be run in production. debug = os.environ.get("DEBUG") if debug: - logging.info("Django is set to use a debugger. Provided debug mode: %s", debug) + logger.info("Django is set to use a debugger. Provided debug mode: %s", debug) if debug == "pycharm-pydevd": - logging.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") + logger.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") import pydevd_pycharm pydevd_pycharm.settrace("host.docker.internal", port=56783, stdoutToServer=True, stderrToServer=True) - logging.info("Debugger started.") + logger.info("Debugger started.") elif debug == "debugpy": - logging.info("Entering debug mode for debugpy (VSCode)") + logger.info("Entering debug mode for debugpy (VSCode)") import debugpy debugpy.listen(("0.0.0.0", 56780)) # noqa S104 (doesn't like binding to all interfaces) - logging.info("Debugger listening on port 56780.") + logger.info("Debugger listening on port 56780.") else: - logging.warning("Invalid debug mode given: %s. Debugger not started", debug) + logger.warning("Invalid debug mode given: %s. Debugger not started", debug) # This allows easy placement of apps within the interior # scram directory. diff --git a/config/consumers.py b/config/consumers.py index 1787707b..76ace3a1 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -8,13 +8,15 @@ from scram.route_manager.models import Entry, WebSocketSequenceElement +logger = logging.getLogger(__name__) + class TranslatorConsumer(AsyncJsonWebsocketConsumer): """Handle messages from the Translator(s).""" async def connect(self): """Handle the initial connection with adding to the right group.""" - logging.info("Translator connected") + logger.info("Translator connected") self.actiontype = self.scope["url_route"]["kwargs"]["actiontype"] self.translator_group = f"translator_{self.actiontype}" @@ -26,7 +28,7 @@ async def connect(self): WebSocketSequenceElement.objects.filter(action_type__name=self.actiontype).order_by("order_num"), ) if not elements: - logging.warning("No elements found for actiontype=%s.", extra=self.actiontype) + logger.warning("No elements found for actiontype=%s.", extra=self.actiontype) return # Avoid lazy evaluation @@ -40,7 +42,7 @@ async def connect(self): async def disconnect(self, close_code): """Discard any remaining messages on disconnect.""" - logging.info("Disconnect received: %s", close_code) + logger.info("Disconnect received: %s", close_code) await self.channel_layer.group_discard(self.translator_group, self.channel_name) async def receive_json(self, content): diff --git a/config/settings/base.py b/config/settings/base.py index 65ad2c95..36960887 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -6,6 +6,8 @@ import environ +logger = logging.getLogger(__name__) + ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent # scram/ APPS_DIR = ROOT_DIR / "scram" @@ -308,7 +310,7 @@ ) OIDC_RP_SIGN_ALGO = "RS256" -logging.info("Using AUTH METHOD=%s", AUTH_METHOD) +logger.info("Using AUTH METHOD=%s", AUTH_METHOD) if AUTH_METHOD == "oidc": # Extend middleware to add OIDC middleware MIDDLEWARE += ["mozilla_django_oidc.middleware.SessionRefresh"] diff --git a/docs/.#summary.md b/docs/.#summary.md new file mode 120000 index 00000000..51b0ff94 --- /dev/null +++ b/docs/.#summary.md @@ -0,0 +1 @@ +vlad@DESKTOP.855 \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index a1699bf4..53f356c5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,7 @@ Security Catch and Release Automation Manager []() []() -[]() +[]() ## Overview diff --git a/manage.py b/manage.py index 6f05512f..cbc80d8d 100755 --- a/manage.py +++ b/manage.py @@ -10,7 +10,7 @@ def main(): """Run administrative tasks.""" os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") try: - from django.core.management import execute_from_command_line + from django.core.management import execute_from_command_line # noqa: PLC0415 except ImportError as exc: msg = ( "Couldn't import Django. Are you sure it's installed and " diff --git a/pyproject.toml b/pyproject.toml index 5bc5458f..5c332104 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,13 +27,13 @@ exclude_also = [ # ===== ruff ==== [tool.ruff] -# Exclude a variety of commonly ignored directories. exclude = [ "migrations", ] line-length = 119 target-version = 'py312' +preview = true [tool.ruff.lint] select = [ @@ -77,6 +77,9 @@ select = [ "UP", # pyupgrade ] ignore = [ + "COM812", # handled by the formatter + "DOC501", # add possible exceptions to the docstring (TODO) + "ISC001", # handled by the formatter "RUF012", # need more widespread typing "SIM102", # Use a single `if` statement instead of nested `if` statements "SIM108", # Use ternary operator instead of `if`-`else`-block @@ -87,6 +90,9 @@ max-complexity = 7 # our current code adheres to this without too much effort [tool.ruff.lint.per-file-ignores] "**/{tests}/*" = [ + "DOC201", # documenting return values + "DOC402", # documenting yield values + "PLR6301", # could be a static method "S101", # use of assert "S106", # hardcoded password ] @@ -94,21 +100,16 @@ max-complexity = 7 # our current code adheres to this without too much effort "S105", # hardcoded password as argument ] "scram/users/**" = [ + "DOC201", # documenting return values "FBT001", # minimal issue; don't need to mess with in the User app "PLR2004", # magic values when checking HTTP status codes ] - -# ==== isort ==== -[tool.isort] -profile = "black" -line_length = 119 -known_first_party = [ - "scram", - "config", +"**/views.py" = [ + "DOC201", # documenting return values; it's fairly obvious in a View ] -skip = ["venv/"] -skip_glob = ["**/migrations/*.py"] +[tool.ruff.lint.pydocstyle] +convention = "google" # ==== mypy ==== [tool.mypy] @@ -130,53 +131,3 @@ ignore_errors = true [tool.django-stubs] django_settings_module = "config.settings.test" - - -# ==== PyLint ==== -[tool.pylint.MASTER] -load-plugins = [ - "pylint_django", -] -django-settings-module = "config.settings.local" - -[tool.pylint.FORMAT] -max-line-length = 119 - -[tool.pylint."MESSAGES CONTROL"] -disable = [ - "missing-docstring", - "invalid-name", -] - -[tool.pylint.DESIGN] -max-parents = 13 - -[tool.pylint.TYPECHECK] -generated-members = [ - "REQUEST", - "acl_users", - "aq_parent", - "[a-zA-Z]+_set{1,2}", - "save", - "delete", -] - - -# ==== djLint ==== -[tool.djlint] -blank_line_after_tag = "load,extends" -close_void_tags = true -format_css = true -format_js = true -# TODO: remove T002 when fixed https://github.com/Riverside-Healthcare/djLint/issues/687 -ignore = "H006,H021,H023,H025,H029,H030,H031,T002,T003" -include = "H017,H035" -indent = 2 -max_line_length = 119 -profile = "django" - -[tool.djlint.css] -indent_size = 2 - -[tool.djlint.js] -indent_size = 2 diff --git a/scram/__init__.py b/scram/__init__.py index d5feb05a..8f5a0220 100644 --- a/scram/__init__.py +++ b/scram/__init__.py @@ -1,4 +1,4 @@ """The Django project for Security Catch and Release Automation Manager (SCRAM).""" __version__ = "1.1.1" -__version_info__ = tuple([int(num) if num.isdigit() else num for num in __version__.replace("-", ".", 1).split(".")]) +__version_info__ = tuple(int(num) if num.isdigit() else num for num in __version__.replace("-", ".", 1).split(".")) diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index a4f6d525..f1bc1907 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -75,19 +75,25 @@ class Meta: model = Entry fields = ["route", "actiontype", "url", "comment", "who"] - def get_comment(self, obj): - """Provide a nicer name for change reason.""" + @staticmethod + def get_comment(obj): + """Provide a nicer name for change reason. + + Returns: + string: The change reason that modified the Entry. + """ return obj.get_change_reason() - def create(self, validated_data): - """Implement custom logic and validates creating a new route.""" + @staticmethod + def create(validated_data): + """Implement custom logic and validates creating a new route.""" # noqa: DOC201 valid_route = validated_data.pop("route") actiontype = validated_data.pop("actiontype") comment = validated_data.pop("comment") - route_instance, created = Route.objects.get_or_create(route=valid_route) + route_instance, _ = Route.objects.get_or_create(route=valid_route) actiontype_instance = ActionType.objects.get(name=actiontype) - entry_instance, created = Entry.objects.get_or_create(route=route_instance, actiontype=actiontype_instance) + entry_instance, _ = Entry.objects.get_or_create(route=route_instance, actiontype=actiontype_instance) logger.debug("Created entry with comment: %s", comment) update_change_reason(entry_instance, comment) diff --git a/scram/route_manager/api/views.py b/scram/route_manager/api/views.py index 802e82d8..70d30b34 100644 --- a/scram/route_manager/api/views.py +++ b/scram/route_manager/api/views.py @@ -20,6 +20,7 @@ from .serializers import ActionTypeSerializer, ClientSerializer, EntrySerializer, IgnoreEntrySerializer channel_layer = get_channel_layer() +logger = logging.getLogger(__name__) @extend_schema( @@ -96,18 +97,19 @@ def check_client_authorization(self, actiontype): ) authorized_client = Client.objects.filter(uuid=uuid).values("is_authorized") if not authorized_client or actiontype not in authorized_actiontypes: - logging.debug("Client: %s, actiontypes: %s", uuid, authorized_actiontypes) - logging.info("%s is not allowed to add an entry to the %s list.", uuid, actiontype) + logger.debug("Client: %s, actiontypes: %s", uuid, authorized_actiontypes) + logger.info("%s is not allowed to add an entry to the %s list.", uuid, actiontype) raise ActiontypeNotAllowed elif not self.request.user.has_perm("route_manager.can_add_entry"): raise PermissionDenied - def check_ignore_list(self, route): + @staticmethod + def check_ignore_list(route): """Ensure that we're not trying to block something from the ignore list.""" overlapping_ignore = IgnoreEntry.objects.filter(route__net_overlaps=route) if overlapping_ignore.count(): ignore_entries = [str(ignore_entry["route"]) for ignore_entry in overlapping_ignore.values()] - logging.info("Cannot proceed adding %s. The ignore list contains %s.", route, ignore_entries) + logger.info("Cannot proceed adding %s. The ignore list contains %s.", route, ignore_entries) raise IgnoredRoute def perform_create(self, serializer): @@ -126,7 +128,7 @@ def perform_create(self, serializer): try: expiration = parse_datetime(tmp_exp) except ValueError: - logging.warning("Could not parse expiration DateTime string: %s", tmp_exp) + logger.warning("Could not parse expiration DateTime string: %s", tmp_exp) # Make sure we put in an acceptable sized prefix min_prefix = getattr(settings, f"V{route.version}_MINPREFIX", 0) @@ -138,7 +140,7 @@ def perform_create(self, serializer): elements = WebSocketSequenceElement.objects.filter(action_type__name=actiontype).order_by("order_num") if not elements: - logging.warning("No elements found for actiontype: %s", actiontype) + logger.warning("No elements found for actiontype: %s", actiontype) for element in elements: msg = element.websocketmessage @@ -157,7 +159,7 @@ def perform_create(self, serializer): entry.who = who entry.is_active = True entry.comment = comment - logging.info("Created entry: %s", entry) + logger.info("Created entry: %s", entry) entry.save() @staticmethod diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index e7793455..d1bafc4e 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -8,7 +8,8 @@ class ESnetAuthBackend(OIDCAuthenticationBackend): """Extend the OIDC backend with a custom permission model.""" - def update_groups(self, user, claims): + @staticmethod + def update_groups(user, claims): """Set the user's group(s) to whatever is in the claims.""" effective_groups = [] claimed_groups = claims.get("groups", []) @@ -28,12 +29,12 @@ def update_groups(self, user, claims): user.save() def create_user(self, claims): - """Wrap the superclass's user creation.""" + """Wrap the superclass's user creation.""" # noqa: DOC201 user = super().create_user(claims) return self.update_user(user, claims) def update_user(self, user, claims): - """Determine the user name from the claims and update said user's groups.""" + """Determine the user name from the claims and update said user's groups.""" # noqa: DOC201 user.name = claims.get("given_name", "") + " " + claims.get("family_name", "") user.username = claims.get("preferred_username", "") if claims.get("groups", False): diff --git a/scram/route_manager/context_processors.py b/scram/route_manager/context_processors.py index 61abde75..0daf06c9 100644 --- a/scram/route_manager/context_processors.py +++ b/scram/route_manager/context_processors.py @@ -5,7 +5,11 @@ def login_logout(request): - """Pass through the relevant URLs from the settings.""" + """Pass through the relevant URLs from the settings. + + Returns: + dict: login and logout URLs + """ login_url = reverse(settings.LOGIN_URL) logout_url = reverse(settings.LOGOUT_URL) return {"login": login_url, "logout": logout_url} diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index 9c123cca..e2267e71 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -10,6 +10,8 @@ from netfields import CidrAddressField from simple_history.models import HistoricalRecords +logger = logging.getLogger(__name__) + class Route(models.Model): """Define a route as a CIDR route and a UUID.""" @@ -18,11 +20,12 @@ class Route(models.Model): uuid = models.UUIDField(db_index=True, default=uuid_lib.uuid4, editable=False) def __str__(self): - """Don't display the UUID, only the route.""" + """Don't display the UUID, only the route.""" # noqa: DOC201 return str(self.route) - def get_absolute_url(self): - """Ensure we use UUID on the API side instead.""" + @staticmethod + def get_absolute_url(): + """Ensure we use UUID on the API side instead.""" # noqa: DOC201 return reverse("") @@ -34,7 +37,7 @@ class ActionType(models.Model): history = HistoricalRecords() def __str__(self): - """Display clearly whether the action is currently available.""" + """Display clearly whether the action is currently available.""" # noqa: DOC201 if not self.available: return f"{self.name} (Inactive)" return self.name @@ -52,7 +55,7 @@ class WebSocketMessage(models.Model): ) def __str__(self): - """Display clearly what the fields are used for.""" + """Display clearly what the fields are used for.""" # noqa: DOC201 return f"{self.msg_type}: {self.msg_data} with the route in key {self.msg_data_route_field}" @@ -76,7 +79,7 @@ class WebSocketSequenceElement(models.Model): action_type = models.ForeignKey("ActionType", on_delete=models.CASCADE) def __str__(self): - """Summarize the fields into something short and readable.""" + """Summarize the fields into something short and readable.""" # noqa: DOC201 return ( f"{self.websocketmessage} as order={self.order_num} for " f"{self.verb} actions on actiontype={self.action_type}" @@ -109,7 +112,7 @@ class Meta: verbose_name_plural = "Entries" def __str__(self): - """Summarize the most important fields to something easily readable.""" + """Summarize the most important fields to something easily readable.""" # noqa: DOC201 desc = f"{self.route} ({self.actiontype})" if not self.is_active: desc += " (inactive)" @@ -121,7 +124,7 @@ def delete(self, *args, **kwargs): # We've already expired this route, don't send another message return # We don't actually delete records; we set them to inactive and then tell the translator to remove them - logging.info("Deactivating %s", self.route) + logger.info("Deactivating %s", self.route) self.is_active = False self.save() @@ -135,7 +138,11 @@ def delete(self, *args, **kwargs): ) def get_change_reason(self): - """Traverse come complex relationships to determine the most recent change reason.""" + """Traverse come complex relationships to determine the most recent change reason. + + Returns: + str: The most recent change reason + """ hist_mgr = getattr(self, self._meta.simple_history_manager_attribute) return hist_mgr.order_by("-history_date").first().history_change_reason @@ -153,7 +160,7 @@ class Meta: verbose_name_plural = "Ignored Entries" def __str__(self): - """Only display the route.""" + """Only display the route.""" # noqa: DOC201 return str(self.route) @@ -167,7 +174,7 @@ class Client(models.Model): authorized_actiontypes = models.ManyToManyField(ActionType) def __str__(self): - """Only display the hostname.""" + """Only display the hostname.""" # noqa: DOC201 return str(self.hostname) diff --git a/scram/route_manager/tests/acceptance/steps/common.py b/scram/route_manager/tests/acceptance/steps/common.py index a81e5a33..f688bd64 100644 --- a/scram/route_manager/tests/acceptance/steps/common.py +++ b/scram/route_manager/tests/acceptance/steps/common.py @@ -21,17 +21,17 @@ def create_actiontype(context, name): {"type": "translator_remove_all", "message": {}}, ) - at, created = ActionType.objects.get_or_create(name=name) - wsm, created = WebSocketMessage.objects.get_or_create(msg_type="translator_add", msg_data_route_field="route") + at, _ = ActionType.objects.get_or_create(name=name) + wsm, _ = WebSocketMessage.objects.get_or_create(msg_type="translator_add", msg_data_route_field="route") wsm.save() - wsse, created = WebSocketSequenceElement.objects.get_or_create(websocketmessage=wsm, verb="A", action_type=at) + wsse, _ = WebSocketSequenceElement.objects.get_or_create(websocketmessage=wsm, verb="A", action_type=at) wsse.save() @given("a client with {name} authorization") def create_authed_client(context, name): """Create a client and authorize it for that action type.""" - at, created = ActionType.objects.get_or_create(name=name) + at, _ = ActionType.objects.get_or_create(name=name) authorized_client = Client.objects.create( hostname="authorized_client.es.net", uuid="0e7e1cbd-7d73-4968-bc4b-ce3265dc2fd3", diff --git a/scram/route_manager/tests/acceptance/steps/translator.py b/scram/route_manager/tests/acceptance/steps/translator.py index adc4812d..ff478a9a 100644 --- a/scram/route_manager/tests/acceptance/steps/translator.py +++ b/scram/route_manager/tests/acceptance/steps/translator.py @@ -10,7 +10,7 @@ async def query_translator(route, actiontype, is_announced): """Ensure the specified route is currently either blocked or unblocked.""" communicator = WebsocketCommunicator(ws_application, f"/ws/route_manager/webui_{actiontype}/") - connected, subprotocol = await communicator.connect() + connected, _ = await communicator.connect() assert connected await communicator.send_json_to({"type": "wui_check_req", "message": {"route": route}}) diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 80260506..fa8b7116 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -154,7 +154,8 @@ class EntryListView(ListView): model = Entry template_name = "route_manager/entry_list.html" - def get_context_data(self, **kwargs): + @staticmethod + def get_context_data(**kwargs): """Group entries by action type.""" context = {"entries": {}} for at in ActionType.objects.all(): diff --git a/scram/users/api/views.py b/scram/users/api/views.py index ce70a713..0e17cb54 100644 --- a/scram/users/api/views.py +++ b/scram/users/api/views.py @@ -23,8 +23,9 @@ def get_queryset(self, *args, **kwargs): """Query on User ID.""" return self.queryset.filter(id=self.request.user.id) + @staticmethod @action(detail=False, methods=["GET"]) - def me(self, request): + def me(request): """Return the current user.""" serializer = UserSerializer(request.user, context={"request": request}) return Response(status=status.HTTP_200_OK, data=serializer.data) diff --git a/scram/users/apps.py b/scram/users/apps.py index 449ebe39..e6cc9c6b 100644 --- a/scram/users/apps.py +++ b/scram/users/apps.py @@ -14,7 +14,8 @@ class UsersConfig(AppConfig): name = "scram.users" verbose_name = _("Users") - def ready(self): + @staticmethod + def ready(): """Check if signals are registered for User events.""" try: import scram.users.signals # noqa F401 diff --git a/scram/users/tests/test_admin.py b/scram/users/tests/test_admin.py index fd49e7ae..4cd32d34 100644 --- a/scram/users/tests/test_admin.py +++ b/scram/users/tests/test_admin.py @@ -15,13 +15,13 @@ def test_changelist(self, admin_client): """Ensure we can view the changelist.""" url = reverse("admin:users_user_changelist") response = admin_client.get(url) - assert response.status_code == 200 + self.assertEqual(response.status_code, 200) def test_search(self, admin_client): """Ensure we can view the search.""" url = reverse("admin:users_user_changelist") response = admin_client.get(url, data={"q": "test"}) - assert response.status_code == 200 + self.assertEqual(response.status_code, 200) def test_add(self, admin_client): """Ensure we can add a user.""" @@ -37,12 +37,12 @@ def test_add(self, admin_client): "password2": "My_R@ndom-P@ssw0rd", }, ) - assert response.status_code == 302 - assert User.objects.filter(username="test").exists() + self.assertEqual(response.status_code, 302) + self.assertTrue(User.objects.filter(username="test").exists()) def test_view_user(self, admin_client): """Ensure we can view a user.""" user = User.objects.get(username="admin") url = reverse("admin:users_user_change", kwargs={"object_id": user.pk}) response = admin_client.get(url) - assert response.status_code == 200 + self.assertEqual(response.status_code, 200) diff --git a/scram/users/tests/test_drf_views.py b/scram/users/tests/test_drf_views.py index bbe56598..29d2afe2 100644 --- a/scram/users/tests/test_drf_views.py +++ b/scram/users/tests/test_drf_views.py @@ -20,7 +20,7 @@ def test_get_queryset(self, user: User, rf: RequestFactory): view.request = request - assert user in view.get_queryset() + self.assertIn(user, view.get_queryset()) def test_me(self, user: User, rf: RequestFactory): """Ensure we can view info on the current user.""" @@ -32,8 +32,11 @@ def test_me(self, user: User, rf: RequestFactory): response = view.me(request) - assert response.data == { - "username": user.username, - "name": user.name, - "url": f"http://testserver/api/v1/users/{user.username}/", - } + self.assertEqual( + response.data, + { + "username": user.username, + "name": user.name, + "url": f"http://testserver/api/v1/users/{user.username}/", + }, + ) diff --git a/scram/users/tests/test_forms.py b/scram/users/tests/test_forms.py index 3fe65da4..574fa24d 100644 --- a/scram/users/tests/test_forms.py +++ b/scram/users/tests/test_forms.py @@ -29,7 +29,7 @@ def test_username_validation_error_msg(self, user: User): }, ) - assert not form.is_valid() - assert len(form.errors) == 1 - assert "username" in form.errors - assert form.errors["username"][0] == _("This username has already been taken.") + self.assertFalse(form.is_valid()) + self.assertEqual(len(form.errors), 1) + self.assertIn("username", form.errors) + self.assertEqual(form.errors["username"][0], _("This username has already been taken.")) diff --git a/scram/users/tests/test_views.py b/scram/users/tests/test_views.py index 2df1929a..c1d5a375 100644 --- a/scram/users/tests/test_views.py +++ b/scram/users/tests/test_views.py @@ -36,7 +36,7 @@ def test_get_success_url(self, user: User, rf: RequestFactory): view.request = request - assert view.get_success_url() == f"/users/{user.username}/" + self.assertEqual(view.get_success_url(), f"/users/{user.username}/") def test_get_object(self, user: User, rf: RequestFactory): """Ensure we can retrieve the User object.""" @@ -46,7 +46,7 @@ def test_get_object(self, user: User, rf: RequestFactory): view.request = request - assert view.get_object() == user + self.assertEqual(view.get_object(), user) def test_form_valid(self, user: User, rf: RequestFactory): """Ensure the form validation works.""" @@ -66,7 +66,7 @@ def test_form_valid(self, user: User, rf: RequestFactory): view.form_valid(form) messages_sent = [m.message for m in messages.get_messages(request)] - assert messages_sent == ["Information successfully updated"] + self.assertEqual(messages_sent, ["Information successfully updated"]) class TestUserRedirectView: @@ -80,7 +80,7 @@ def test_get_redirect_url(self, user: User, rf: RequestFactory): view.request = request - assert view.get_redirect_url() == f"/users/{user.username}/" + self.assertEqual(view.get_redirect_url(), f"/users/{user.username}/") class TestUserDetailView: @@ -93,7 +93,7 @@ def test_authenticated(self, user: User, rf: RequestFactory): response = user_detail_view(request, username=user.username) - assert response.status_code == 200 + self.assertEqual(response.status_code, 200) def test_not_authenticated(self, user: User, rf: RequestFactory): """Ensure an unauthenticated user gets redirected to login.""" @@ -103,5 +103,5 @@ def test_not_authenticated(self, user: User, rf: RequestFactory): response = user_detail_view(request, username=user.username) login_url = reverse(settings.LOGIN_URL) - assert response.status_code == 302 - assert response.url == f"{login_url}?next=/fake-url/" + self.assertEqual(response.status_code, 302) + self.assertEqual(response.url, f"{login_url}?next=/fake-url/") diff --git a/scram/utils/context_processors.py b/scram/utils/context_processors.py index 2a86dfa4..0bbf8786 100644 --- a/scram/utils/context_processors.py +++ b/scram/utils/context_processors.py @@ -4,7 +4,11 @@ def settings_context(_request): - """Define settings available by default to the templates context.""" + """Define settings available by default to the templates context. + + Returns: + dict: Whether or not we have DEBUG on + """ # Note: we intentionally do NOT expose the entire settings # to prevent accidental leaking of sensitive information return {"DEBUG": settings.DEBUG} diff --git a/translator/gobgp.py b/translator/gobgp.py index e67a92d1..f2b8dee9 100644 --- a/translator/gobgp.py +++ b/translator/gobgp.py @@ -20,6 +20,7 @@ IPV6 = 6 logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) class GoBGP: @@ -30,12 +31,13 @@ def __init__(self, url): channel = grpc.insecure_channel(url) self.stub = gobgp_pb2_grpc.GobgpApiStub(channel) - def _get_family_afi(self, ip_version): + @staticmethod + def _get_family_afi(ip_version): if ip_version == IPV6: return gobgp_pb2.Family.AFI_IP6 return gobgp_pb2.Family.AFI_IP - def _build_path(self, ip, event_data=None): + def _build_path(self, ip, event_data=None): # noqa: PLR0914 # Grab ASN and Community from our event_data, or use the defaults if not event_data: event_data = {} @@ -104,7 +106,7 @@ def _build_path(self, ip, event_data=None): comm_id = (asn << 16) + community communities.Pack(attribute_pb2.CommunitiesAttribute(communities=[comm_id])) else: - logging.info("LargeCommunity Used - ASN: %s. Community: %s", asn, community) + logger.info("LargeCommunity Used - ASN: %s. Community: %s", asn, community) global_admin = asn local_data1 = community # set to 0 because there's no use case for it, but we need a local_data2 for gobgp to read any of it @@ -126,7 +128,7 @@ def _build_path(self, ip, event_data=None): def add_path(self, ip, event_data): """Announce a single route.""" - logging.info("Blocking %s", ip) + logger.info("Blocking %s", ip) try: path = self._build_path(ip, event_data) @@ -135,17 +137,17 @@ def add_path(self, ip, event_data): _TIMEOUT_SECONDS, ) except ASNError as e: - logging.warning("ASN assertion failed with error: %s", e) + logger.warning("ASN assertion failed with error: %s", e) def del_all_paths(self): """Remove all routes from being announced.""" - logging.warning("Withdrawing ALL routes") + logger.warning("Withdrawing ALL routes") self.stub.DeletePath(gobgp_pb2.DeletePathRequest(table_type=gobgp_pb2.GLOBAL), _TIMEOUT_SECONDS) def del_path(self, ip, event_data): """Remove a single route from being announced.""" - logging.info("Unblocking %s", ip) + logger.info("Unblocking %s", ip) try: path = self._build_path(ip, event_data) self.stub.DeletePath( @@ -153,10 +155,14 @@ def del_path(self, ip, event_data): _TIMEOUT_SECONDS, ) except ASNError as e: - logging.warning("ASN assertion failed with error: %s", e) + logger.warning("ASN assertion failed with error: %s", e) def get_prefixes(self, ip): - """Retrieve the routes that match a prefix and are announced.""" + """Retrieve the routes that match a prefix and are announced. + + Returns: + list: The routes that overlap with the prefix and are currently announced. + """ prefixes = [gobgp_pb2.TableLookupPrefix(prefix=str(ip.ip))] family_afi = self._get_family_afi(ip.ip.version) result = self.stub.ListPath( diff --git a/translator/tests/acceptance/steps/actions.py b/translator/tests/acceptance/steps/actions.py index 83526b27..fdb2a95f 100644 --- a/translator/tests/acceptance/steps/actions.py +++ b/translator/tests/acceptance/steps/actions.py @@ -27,7 +27,11 @@ def del_block(context, route, asn, community): def get_block_status(context, ip): - """Check if the IP is currently blocked.""" + """Check if the IP is currently blocked. + + Returns: + bool: The return value. True if the IP is currently blocked, False otherwise. + """ # Allow our add/delete requests to settle time.sleep(1) diff --git a/translator/translator.py b/translator/translator.py index ef168c60..3cba28c8 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -11,6 +11,8 @@ import websockets from gobgp import GoBGP +logger = logging.getLogger(__name__) + # Here we setup a debugger if this is desired. This obviously should not be run in production. debug_mode = os.environ.get("DEBUG") if debug_mode: @@ -23,20 +25,20 @@ def install_deps(): which is unecessary, but in the future when we build a little better, it'll already be setup. """ - logging.info("Installing dependencies for debuggers") + logger.info("Installing dependencies for debuggers") - import subprocess - import sys + import subprocess # noqa: S404, PLC0415 + import sys # noqa: PLC0415 subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "/requirements/local.txt"]) # noqa: S603 TODO: add this to the container build - logging.info("Done installing dependencies for debuggers") + logger.info("Done installing dependencies for debuggers") - logging.info("Translator is set to use a debugger. Provided debug mode: %s", debug_mode) + logger.info("Translator is set to use a debugger. Provided debug mode: %s", debug_mode) # We have to setup the debugger appropriately for various IDEs. It'd be nice if they all used the same thing but # sadly, we live in a fallen world. if debug_mode == "pycharm-pydevd": - logging.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") + logger.info("Entering debug mode for pycharm, make sure the debug server is running in PyCharm!") install_deps() @@ -44,9 +46,9 @@ def install_deps(): pydevd_pycharm.settrace("host.docker.internal", port=56782, stdoutToServer=True, stderrToServer=True) - logging.info("Debugger started.") + logger.info("Debugger started.") elif debug_mode == "debugpy": - logging.info("Entering debug mode for debugpy (VSCode)") + logger.info("Entering debug mode for debugpy (VSCode)") install_deps() @@ -54,9 +56,9 @@ def install_deps(): debugpy.listen(("0.0.0.0", 56781)) # noqa S104 (doesn't like binding to all interfaces) - logging.info("Debugger listening on port 56781.") + logger.info("Debugger listening on port 56781.") else: - logging.warning("Invalid debug mode given: %s. Debugger not started", debug_mode) + logger.warning("Invalid debug mode given: %s. Debugger not started", debug_mode) # Must match the URL in asgi.py, and needs a trailing slash hostname = os.environ.get("SCRAM_HOSTNAME", "scram_hostname_not_set") @@ -69,7 +71,7 @@ def install_deps(): ] -async def block_or_unblock(gobgp, event_type, event_data, ip): +def block_or_unblock(gobgp, event_type, event_data, ip): """Receive one message and pass it off to the right function.""" # TODO: Maybe only allow this in testing? if event_type == "translator_remove_all": @@ -92,13 +94,13 @@ async def main(): event_type = raw_message.get("type") event_data = raw_message.get("message") if event_type not in KNOWN_MESSAGES: - logging.error("Unknown event type received: %s", event_type) + logger.error("Unknown event type received: %s", event_type) continue try: ip = ipaddress.ip_interface(event_data["route"]) except: # noqa E722 - logging.exception("Error parsing message: %s", raw_message) + logger.exception("Error parsing message: %s", raw_message) continue if event_type == "translator_check": @@ -114,7 +116,7 @@ async def main(): if __name__ == "__main__": - logging.info("translator started") + logger.info("translator started") loop = asyncio.get_event_loop() loop.run_until_complete(main()) loop.close() From 4a2693909d58e9e831646714c71f5351ac4790c0 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 14:52:44 -0600 Subject: [PATCH 119/156] Fix auth simplification logic --- scram/route_manager/authentication_backends.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index d1bafc4e..719b8577 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -5,6 +5,15 @@ from mozilla_django_oidc.auth import OIDCAuthenticationBackend +def groups_overlap(a, b): + """Helper function to see if a and b have any overlap. + + Returns: + bool: True if there's any overlap between a and b. + """ + return not set(a).isdisjoint(b) + + class ESnetAuthBackend(OIDCAuthenticationBackend): """Extend the OIDC backend with a custom permission model.""" @@ -14,14 +23,14 @@ def update_groups(user, claims): effective_groups = [] claimed_groups = claims.get("groups", []) - if any(claimed_groups in settings.SCRAM_DENIED_GROUPS): + if groups_overlap(claimed_groups, settings.SCRAM_DENIED_GROUPS): is_admin = False # Don't even look at anything else if they're denied else: - is_admin = any(claimed_groups in settings.SCRAM_ADMIN_GROUPS) - if any(claimed_groups in settings.SCRAM_READWRITE_GROUPS): + is_admin = groups_overlap(claimed_groups, settings.SCRAM_ADMIN_GROUPS) + if groups_overlap(claimed_groups, settings.SCRAM_READWRITE_GROUPS): effective_groups += Group.objects.get(name="readwrite") - if any(claimed_groups in settings.SCRAM_READONLY_GROUPS): + if groups_overlap(claimed_groups, settings.SCRAM_READONLY_GROUPS): effective_groups += Group.objects.get(name="readonly") user.groups.set(effective_groups) From 9d7831a26f184c1022ed93e03845f7294d8d29e1 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 15:24:18 -0600 Subject: [PATCH 120/156] Fix a few failed tests --- .github/workflows/ruff.yml | 2 +- scram/route_manager/authentication_backends.py | 4 ++-- scram/route_manager/tests/test_authorization.py | 2 +- scram/users/tests/test_admin.py | 10 +++++----- scram/users/tests/test_forms.py | 8 ++++---- scram/users/tests/test_views.py | 14 +++++++------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 2a58b8bd..31647732 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -23,7 +23,7 @@ jobs: run: pip install ruff - name: Fail if we have any style errors - run: ruff check + run: ruff check --output-format=github - name: Fail if the code is not formatted correctly (like Black) run: ruff format --diff diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index 719b8577..1831d788 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -29,9 +29,9 @@ def update_groups(user, claims): else: is_admin = groups_overlap(claimed_groups, settings.SCRAM_ADMIN_GROUPS) if groups_overlap(claimed_groups, settings.SCRAM_READWRITE_GROUPS): - effective_groups += Group.objects.get(name="readwrite") + effective_groups.append(Group.objects.get(name="readwrite")) if groups_overlap(claimed_groups, settings.SCRAM_READONLY_GROUPS): - effective_groups += Group.objects.get(name="readonly") + effective_groups.append(Group.objects.get(name="readonly")) user.groups.set(effective_groups) user.is_staff = user.is_superuser = is_admin diff --git a/scram/route_manager/tests/test_authorization.py b/scram/route_manager/tests/test_authorization.py index 17de2b6d..9a0d4bff 100644 --- a/scram/route_manager/tests/test_authorization.py +++ b/scram/route_manager/tests/test_authorization.py @@ -236,7 +236,7 @@ def test_authorized_removal(self): def test_disabled(self): """Pass all the groups, user should be disabled as it takes precedence.""" claims = dict(self.claims) - claims["groups"] = [settings.SCRAM_GROUPS] + claims["groups"] = settings.SCRAM_GROUPS user = ESnetAuthBackend().create_user(claims) self.assertFalse(user.is_staff) diff --git a/scram/users/tests/test_admin.py b/scram/users/tests/test_admin.py index 4cd32d34..fd49e7ae 100644 --- a/scram/users/tests/test_admin.py +++ b/scram/users/tests/test_admin.py @@ -15,13 +15,13 @@ def test_changelist(self, admin_client): """Ensure we can view the changelist.""" url = reverse("admin:users_user_changelist") response = admin_client.get(url) - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 def test_search(self, admin_client): """Ensure we can view the search.""" url = reverse("admin:users_user_changelist") response = admin_client.get(url, data={"q": "test"}) - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 def test_add(self, admin_client): """Ensure we can add a user.""" @@ -37,12 +37,12 @@ def test_add(self, admin_client): "password2": "My_R@ndom-P@ssw0rd", }, ) - self.assertEqual(response.status_code, 302) - self.assertTrue(User.objects.filter(username="test").exists()) + assert response.status_code == 302 + assert User.objects.filter(username="test").exists() def test_view_user(self, admin_client): """Ensure we can view a user.""" user = User.objects.get(username="admin") url = reverse("admin:users_user_change", kwargs={"object_id": user.pk}) response = admin_client.get(url) - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 diff --git a/scram/users/tests/test_forms.py b/scram/users/tests/test_forms.py index 574fa24d..3fe65da4 100644 --- a/scram/users/tests/test_forms.py +++ b/scram/users/tests/test_forms.py @@ -29,7 +29,7 @@ def test_username_validation_error_msg(self, user: User): }, ) - self.assertFalse(form.is_valid()) - self.assertEqual(len(form.errors), 1) - self.assertIn("username", form.errors) - self.assertEqual(form.errors["username"][0], _("This username has already been taken.")) + assert not form.is_valid() + assert len(form.errors) == 1 + assert "username" in form.errors + assert form.errors["username"][0] == _("This username has already been taken.") diff --git a/scram/users/tests/test_views.py b/scram/users/tests/test_views.py index c1d5a375..2df1929a 100644 --- a/scram/users/tests/test_views.py +++ b/scram/users/tests/test_views.py @@ -36,7 +36,7 @@ def test_get_success_url(self, user: User, rf: RequestFactory): view.request = request - self.assertEqual(view.get_success_url(), f"/users/{user.username}/") + assert view.get_success_url() == f"/users/{user.username}/" def test_get_object(self, user: User, rf: RequestFactory): """Ensure we can retrieve the User object.""" @@ -46,7 +46,7 @@ def test_get_object(self, user: User, rf: RequestFactory): view.request = request - self.assertEqual(view.get_object(), user) + assert view.get_object() == user def test_form_valid(self, user: User, rf: RequestFactory): """Ensure the form validation works.""" @@ -66,7 +66,7 @@ def test_form_valid(self, user: User, rf: RequestFactory): view.form_valid(form) messages_sent = [m.message for m in messages.get_messages(request)] - self.assertEqual(messages_sent, ["Information successfully updated"]) + assert messages_sent == ["Information successfully updated"] class TestUserRedirectView: @@ -80,7 +80,7 @@ def test_get_redirect_url(self, user: User, rf: RequestFactory): view.request = request - self.assertEqual(view.get_redirect_url(), f"/users/{user.username}/") + assert view.get_redirect_url() == f"/users/{user.username}/" class TestUserDetailView: @@ -93,7 +93,7 @@ def test_authenticated(self, user: User, rf: RequestFactory): response = user_detail_view(request, username=user.username) - self.assertEqual(response.status_code, 200) + assert response.status_code == 200 def test_not_authenticated(self, user: User, rf: RequestFactory): """Ensure an unauthenticated user gets redirected to login.""" @@ -103,5 +103,5 @@ def test_not_authenticated(self, user: User, rf: RequestFactory): response = user_detail_view(request, username=user.username) login_url = reverse(settings.LOGIN_URL) - self.assertEqual(response.status_code, 302) - self.assertEqual(response.url, f"{login_url}?next=/fake-url/") + assert response.status_code == 302 + assert response.url == f"{login_url}?next=/fake-url/" From cb96b2c3e41791317f6af1b2aae22cc7c94467cc Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 15:35:06 -0600 Subject: [PATCH 121/156] Fix another test, and silence a pip warning in Docker --- compose/local/django/.#start | 1 + compose/local/django/Dockerfile | 1 + compose/local/docs/Dockerfile | 1 + scram/users/tests/test_drf_views.py | 15 ++++++--------- 4 files changed, 9 insertions(+), 9 deletions(-) create mode 120000 compose/local/django/.#start diff --git a/compose/local/django/.#start b/compose/local/django/.#start new file mode 120000 index 00000000..efa1825f --- /dev/null +++ b/compose/local/django/.#start @@ -0,0 +1 @@ +vlad@DESKTOP.796 \ No newline at end of file diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 64bd4a6b..12ea411b 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.12-slim-bookworm +ENV PIP_ROOT_USER_ACTION ignore ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/compose/local/docs/Dockerfile b/compose/local/docs/Dockerfile index 758ca263..1aedf840 100644 --- a/compose/local/docs/Dockerfile +++ b/compose/local/docs/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.12-slim-bookworm +ENV PIP_ROOT_USER_ACTION ignore ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/scram/users/tests/test_drf_views.py b/scram/users/tests/test_drf_views.py index 29d2afe2..bbe56598 100644 --- a/scram/users/tests/test_drf_views.py +++ b/scram/users/tests/test_drf_views.py @@ -20,7 +20,7 @@ def test_get_queryset(self, user: User, rf: RequestFactory): view.request = request - self.assertIn(user, view.get_queryset()) + assert user in view.get_queryset() def test_me(self, user: User, rf: RequestFactory): """Ensure we can view info on the current user.""" @@ -32,11 +32,8 @@ def test_me(self, user: User, rf: RequestFactory): response = view.me(request) - self.assertEqual( - response.data, - { - "username": user.username, - "name": user.name, - "url": f"http://testserver/api/v1/users/{user.username}/", - }, - ) + assert response.data == { + "username": user.username, + "name": user.name, + "url": f"http://testserver/api/v1/users/{user.username}/", + } From d1c57b1000c5282b09fdf4baadf1164956256be2 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 17:09:56 -0600 Subject: [PATCH 122/156] Reducing complexity broke xlator behave --- pyproject.toml | 2 +- translator/translator.py | 64 ++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5c332104..684b4e2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ ignore = [ ] [tool.ruff.lint.mccabe] -max-complexity = 7 # our current code adheres to this without too much effort +#max-complexity = 7 # our current code adheres to this without too much effort [tool.ruff.lint.per-file-ignores] "**/{tests}/*" = [ diff --git a/translator/translator.py b/translator/translator.py index 3cba28c8..202d986f 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -13,6 +13,13 @@ logger = logging.getLogger(__name__) +KNOWN_MESSAGES = { + "translator_add", + "translator_remove", + "translator_remove_all", + "translator_check", +} + # Here we setup a debugger if this is desired. This obviously should not be run in production. debug_mode = os.environ.get("DEBUG") if debug_mode: @@ -63,25 +70,6 @@ def install_deps(): # Must match the URL in asgi.py, and needs a trailing slash hostname = os.environ.get("SCRAM_HOSTNAME", "scram_hostname_not_set") url = os.environ.get("SCRAM_EVENTS_URL", "ws://django:8000/ws/route_manager/translator_block/") -KNOWN_MESSAGES = [ - "translator_add", - "translator_remove", - "translator_remove_all", - "translator_check", -] - - -def block_or_unblock(gobgp, event_type, event_data, ip): - """Receive one message and pass it off to the right function.""" - # TODO: Maybe only allow this in testing? - if event_type == "translator_remove_all": - gobgp.del_all_paths() - return - - if event_type == "translator_add": - gobgp.add_path(ip, event_data) - elif event_type == "translator_remove": - gobgp.del_path(ip, event_data) async def main(): @@ -90,26 +78,30 @@ async def main(): async for websocket in websockets.connect(url): try: async for message in websocket: - raw_message = json.loads(message) - event_type = raw_message.get("type") - event_data = raw_message.get("message") + json_message = json.loads(message) + event_type = json_message.get("type") + event_data = json_message.get("message") if event_type not in KNOWN_MESSAGES: logger.error("Unknown event type received: %s", event_type) - continue - - try: - ip = ipaddress.ip_interface(event_data["route"]) - except: # noqa E722 - logger.exception("Error parsing message: %s", raw_message) - continue - - if event_type == "translator_check": - raw_message["type"] = "translator_check_resp" - raw_message["message"]["is_blocked"] = g.is_blocked(ip) - raw_message["message"]["translator_name"] = hostname - await websocket.send(json.dumps(raw_message)) + # TODO: Maybe only allow this in testing? + elif event_type == "translator_remove_all": + g.del_all_paths() else: - await block_or_unblock(g, event_type, event_data, ip) + try: + ip = ipaddress.ip_interface(event_data["route"]) + except: # noqa E722 + logger.exception("Error parsing message: %s", message) + continue + + if event_type == "translator_add": + g.add_path(ip, event_data) + elif event_type == "translator_remove": + g.del_path(ip, event_data) + elif event_type == "translator_check": + json_message["type"] = "translator_check_resp" + json_message["message"]["is_blocked"] = g.is_blocked(ip) + json_message["message"]["translator_name"] = hostname + await websocket.send(json.dumps(json_message)) except websockets.ConnectionClosed: continue From 42f6a84ef4db636771b829dbf4df4013a9e23c72 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 17:22:29 -0600 Subject: [PATCH 123/156] Refactor xlator code a bit to reduce complexity --- pyproject.toml | 2 +- translator/translator.py | 53 ++++++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 684b4e2d..5c332104 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,7 @@ ignore = [ ] [tool.ruff.lint.mccabe] -#max-complexity = 7 # our current code adheres to this without too much effort +max-complexity = 7 # our current code adheres to this without too much effort [tool.ruff.lint.per-file-ignores] "**/{tests}/*" = [ diff --git a/translator/translator.py b/translator/translator.py index 202d986f..d2c34018 100644 --- a/translator/translator.py +++ b/translator/translator.py @@ -72,36 +72,41 @@ def install_deps(): url = os.environ.get("SCRAM_EVENTS_URL", "ws://django:8000/ws/route_manager/translator_block/") +async def process(message, websocket, g): + """Take a single message form the websocket and hand it off to the appropriate function.""" + json_message = json.loads(message) + event_type = json_message.get("type") + event_data = json_message.get("message") + if event_type not in KNOWN_MESSAGES: + logger.error("Unknown event type received: %s", event_type) + # TODO: Maybe only allow this in testing? + elif event_type == "translator_remove_all": + g.del_all_paths() + else: + try: + ip = ipaddress.ip_interface(event_data["route"]) + except: # noqa E722 + logger.exception("Error parsing message: %s", message) + return + + if event_type == "translator_add": + g.add_path(ip, event_data) + elif event_type == "translator_remove": + g.del_path(ip, event_data) + elif event_type == "translator_check": + json_message["type"] = "translator_check_resp" + json_message["message"]["is_blocked"] = g.is_blocked(ip) + json_message["message"]["translator_name"] = hostname + await websocket.send(json.dumps(json_message)) + + async def main(): """Connect to the websocket and start listening for messages.""" g = GoBGP("gobgp:50051") async for websocket in websockets.connect(url): try: async for message in websocket: - json_message = json.loads(message) - event_type = json_message.get("type") - event_data = json_message.get("message") - if event_type not in KNOWN_MESSAGES: - logger.error("Unknown event type received: %s", event_type) - # TODO: Maybe only allow this in testing? - elif event_type == "translator_remove_all": - g.del_all_paths() - else: - try: - ip = ipaddress.ip_interface(event_data["route"]) - except: # noqa E722 - logger.exception("Error parsing message: %s", message) - continue - - if event_type == "translator_add": - g.add_path(ip, event_data) - elif event_type == "translator_remove": - g.del_path(ip, event_data) - elif event_type == "translator_check": - json_message["type"] = "translator_check_resp" - json_message["message"]["is_blocked"] = g.is_blocked(ip) - json_message["message"]["translator_name"] = hostname - await websocket.send(json.dumps(json_message)) + await process(message, websocket, g) except websockets.ConnectionClosed: continue From e78323565c9f64315b0d2035ed6ca665ff4f77f4 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 17:35:39 -0600 Subject: [PATCH 124/156] Add a workflow that runs pytest outside of containers. For speed. --- .github/workflows/behave.yml | 83 ++++++++++++++++++++++++++++++++++++ .github/workflows/pytest.yml | 72 +++++++++++++++---------------- 2 files changed, 117 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/behave.yml diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml new file mode 100644 index 00000000..ec12d633 --- /dev/null +++ b/.github/workflows/behave.yml @@ -0,0 +1,83 @@ +--- +name: Run behave + +on: + push: + branches: + - '**' + pull_request: + branches: + - main + - develop + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + behave: + name: Run Behave + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: scram + POSTGRES_PASSWORD: '' + POSTGRES_DB: test_scram + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U scram" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose make + + - name: Build Docker images + run: make build + + - name: Migrate Database + run: make migrate + + - name: Run Application + run: make run + + - name: Run pytest + behave with Coverage + env: + POSTGRES_USER: scram + POSTGRES_DB: test_scram + run: make coverage.xml + + - name: Upload Coverage to Coveralls + uses: coverallsapp/github-action@v2 + + - name: Upload Coverage to GitHub + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: coverage.xml + + - name: Display Coverage Metrics + uses: 5monkeys/cobertura-action@v14 + with: + minimum_coverage: '50' + + - name: Stop Services + if: always() + run: make stop + + - name: Clean Up + if: always() + run: make clean diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f84957df..39d1cc2a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,6 +17,10 @@ jobs: pytest: name: Run Pytest runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: [3.9, 3.10, 3.11, 3.12] services: postgres: @@ -38,46 +42,38 @@ jobs: - name: Check out the code uses: actions/checkout@v4 - - name: Set up Docker - uses: docker/setup-buildx-action@v3 - - name: Install Docker Compose - run: | - sudo apt-get update - sudo apt-get install -y docker-compose make - - - name: Build Docker images - run: make build - - - name: Migrate Database - run: make migrate + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - - name: Run Application - run: make run + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements/local.txt + pip install pytest-github-actions-annotate-failures - - name: Run Pytest with Coverage + - name: Apply unapplied migrations env: - POSTGRES_USER: scram - POSTGRES_DB: test_scram - run: make coverage.xml - - - name: Upload Coverage to Coveralls - uses: coverallsapp/github-action@v2 - - - name: Upload Coverage to GitHub - uses: actions/upload-artifact@v4 - with: - name: coverage-report - path: coverage.xml - - - name: Display Coverage Metrics - uses: 5monkeys/cobertura-action@v14 - with: - minimum_coverage: '50' + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + run: | + python manage.py makemigrations --noinput || true + UNAPPLIED_MIGRATIONS=$(python manage.py showmigrations --plan | grep '\[ \]' | awk '{print $2}') + if [ -n "$UNAPPLIED_MIGRATIONS" ]; then + for migration in $UNAPPLIED_MIGRATIONS; do + python manage.py migrate $migration --fake-initial --noinput + done + else + echo "No unapplied migrations." + fi - - name: Stop Services - if: always() - run: make stop + - name: Check for duplicate migrations + run: | + if python manage.py makemigrations --dry-run | grep "No changes detected"; then + echo "No duplicate migrations detected." + else + echo "Warning: Potential duplicate migrations detected. Please review." - - name: Clean Up - if: always() - run: make clean + - name: Run Pytest + env: + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + run: pytest \ No newline at end of file From 037a7f206c90e36decdedc889ff69b6461eb912d Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 17:37:58 -0600 Subject: [PATCH 125/156] Change Python matrix --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 39d1cc2a..129af047 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,7 +20,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.9, 3.10, 3.11, 3.12] + python-version: ['3.11', '3.12', '3.13'] services: postgres: From b24460338e54ad8a11a74a9f87e4cce74c977f18 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 17:45:08 -0600 Subject: [PATCH 126/156] Fix bash bug --- .github/workflows/pytest.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 129af047..b946d06c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -42,7 +42,7 @@ jobs: - name: Check out the code uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -72,8 +72,9 @@ jobs: echo "No duplicate migrations detected." else echo "Warning: Potential duplicate migrations detected. Please review." + fi - name: Run Pytest env: DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" - run: pytest \ No newline at end of file + run: pytest From 89e828a058b41a2fdeaa85bd79ac3790fb466199 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 17:55:35 -0600 Subject: [PATCH 127/156] Add an env var for the Redis host. --- .github/workflows/pytest.yml | 6 ++++++ config/settings/base.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b946d06c..9ac8bf25 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -38,6 +38,11 @@ jobs: --health-timeout 5s --health-retries 5 + redis: + image: redis:5.0 + ports: + - 6379:6379 + steps: - name: Check out the code uses: actions/checkout@v4 @@ -77,4 +82,5 @@ jobs: - name: Run Pytest env: DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + REDIS_HOST: "localhost" run: pytest diff --git a/config/settings/base.py b/config/settings/base.py index 36960887..be90d71e 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -256,7 +256,7 @@ "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { - "hosts": [("redis", 6379)], + "hosts": [(os.environ.get("REDIS_HOST", "redis"), 6379)], }, }, } From ecdf33149f1a7ee1555cc18d7b401cf873378a0f Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 18:03:47 -0600 Subject: [PATCH 128/156] Add a test that is allowed to fail to test a future Python version. --- .github/workflows/future_pytest.yml | 87 +++++++++++++++++++++++++++++ .github/workflows/pytest.yml | 2 +- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/future_pytest.yml diff --git a/.github/workflows/future_pytest.yml b/.github/workflows/future_pytest.yml new file mode 100644 index 00000000..58e1803b --- /dev/null +++ b/.github/workflows/future_pytest.yml @@ -0,0 +1,87 @@ +--- +name: Run pytest with unsupported Python versions + +on: + push: + branches: + - '**' + pull_request: + branches: + - main + - develop + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + pytest: + name: Run Pytest + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: ['3.13'] + + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: scram + POSTGRES_PASSWORD: '' + POSTGRES_DB: test_scram + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U scram" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + redis: + image: redis:5.0 + ports: + - 6379:6379 + + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements/local.txt + pip install pytest-github-actions-annotate-failures + + - name: Apply unapplied migrations + env: + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + run: | + python manage.py makemigrations --noinput || true + UNAPPLIED_MIGRATIONS=$(python manage.py showmigrations --plan | grep '\[ \]' | awk '{print $2}') + if [ -n "$UNAPPLIED_MIGRATIONS" ]; then + for migration in $UNAPPLIED_MIGRATIONS; do + python manage.py migrate $migration --fake-initial --noinput + done + else + echo "No unapplied migrations." + fi + + - name: Check for duplicate migrations + run: | + if python manage.py makemigrations --dry-run | grep "No changes detected"; then + echo "No duplicate migrations detected." + else + echo "Warning: Potential duplicate migrations detected. Please review." + fi + + - name: Run Pytest + env: + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + REDIS_HOST: "localhost" + run: | + pytest || echo "::warning:: Failed on future Python version ${{ matrix.python-version }}." diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 9ac8bf25..b4c23339 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,7 +20,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.11', '3.12', '3.13'] + python-version: ['3.11', '3.12'] services: postgres: From ef4dd1f220a483b9919b22f2ad26f02f99325d93 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 18:11:25 -0600 Subject: [PATCH 129/156] Running multiple was causing an asyncio conflict. --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b4c23339..45feca20 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,7 +20,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.11', '3.12'] + python-version: ['3.12'] services: postgres: From 136141e2af23237343ddccf1331d4a7d7c17446d Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 18:32:51 -0600 Subject: [PATCH 130/156] Final fixes --- compose/local/django/.#start | 1 - config/consumers.py | 2 +- scram/route_manager/models.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) delete mode 120000 compose/local/django/.#start diff --git a/compose/local/django/.#start b/compose/local/django/.#start deleted file mode 120000 index efa1825f..00000000 --- a/compose/local/django/.#start +++ /dev/null @@ -1 +0,0 @@ -vlad@DESKTOP.796 \ No newline at end of file diff --git a/config/consumers.py b/config/consumers.py index 76ace3a1..cd163bb7 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -28,7 +28,7 @@ async def connect(self): WebSocketSequenceElement.objects.filter(action_type__name=self.actiontype).order_by("order_num"), ) if not elements: - logger.warning("No elements found for actiontype=%s.", extra=self.actiontype) + logger.warning("No elements found for actiontype=%s.", self.actiontype) return # Avoid lazy evaluation diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index e2267e71..aa72f7ba 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -138,7 +138,7 @@ def delete(self, *args, **kwargs): ) def get_change_reason(self): - """Traverse come complex relationships to determine the most recent change reason. + """Traverse some complex relationships to determine the most recent change reason. Returns: str: The most recent change reason From 6a4c8e2ca9225424262a193348fbe3bd0ecd4eda Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 19:46:39 -0600 Subject: [PATCH 131/156] Add a few meaningful tests to the homepage --- scram/route_manager/tests/test_views.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scram/route_manager/tests/test_views.py b/scram/route_manager/tests/test_views.py index bcc3a4ad..ee9e9e1c 100644 --- a/scram/route_manager/tests/test_views.py +++ b/scram/route_manager/tests/test_views.py @@ -1,7 +1,7 @@ """Define simple tests for the template-based Views.""" from django.test import TestCase -from django.urls import resolve +from django.urls import resolve, reverse from scram.route_manager.views import home_page @@ -13,3 +13,19 @@ def test_root_url_resolves_to_home_page_view(self): """Ensure we can find the home page.""" found = resolve("/") self.assertEqual(found.func, home_page) + + +class HomePageFirstVisitTest(TestCase): + """Test how the home page renders the first time we view it.""" + + def setUp(self): + """Get the home page.""" + self.response = self.client.get(reverse("route_manager:home")) + + def test_first_homepage_view_has_userinfo(self): + """The first time we view the home page, a user was created for us.""" + self.assertContains(self.response, b"An admin user was created for you.") + + def test_first_homepage_view_is_logged_in(self): + """The first time we view the home page, we're logged in.""" + self.assertContains(self.response, b'type="submit">Logout') \ No newline at end of file From d8f5cb1cd2e87c841dd1d7cd64858e0a2a572a5d Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 20:44:31 -0600 Subject: [PATCH 132/156] Add a few more tests for the home page --- scram/route_manager/tests/test_views.py | 30 ++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/scram/route_manager/tests/test_views.py b/scram/route_manager/tests/test_views.py index ee9e9e1c..ae46a5fa 100644 --- a/scram/route_manager/tests/test_views.py +++ b/scram/route_manager/tests/test_views.py @@ -1,5 +1,6 @@ """Define simple tests for the template-based Views.""" +from django.conf import settings from django.test import TestCase from django.urls import resolve, reverse @@ -28,4 +29,31 @@ def test_first_homepage_view_has_userinfo(self): def test_first_homepage_view_is_logged_in(self): """The first time we view the home page, we're logged in.""" - self.assertContains(self.response, b'type="submit">Logout') \ No newline at end of file + self.assertContains(self.response, b'type="submit">Logout') + + +class HomePageLogoutTest(TestCase): + """Verify that once logged out, we can't view anything.""" + + def test_homepage_logout_links_missing(self): + """After logout, we can't see anything.""" + response = self.client.get(reverse("route_manager:home")) + response = self.client.post(reverse(settings.LOGOUT_URL), follow=True) + self.assertEqual(response.status_code, 200) + response = self.client.get(reverse("route_manager:home")) + print(response.content) + + self.assertNotContains(response, b"An admin user was created for you.") + self.assertNotContains(response, b'type="submit">Logout') + self.assertNotContains(response, b">Admin") + + +class NotFoundTest(TestCase): + """Verify that our custom 404 page is being served up.""" + + def test_404(self): + """Grab a bad URL.""" + response = self.client.get("/foobarbaz") + self.assertContains( + response, b'
The page you are looking for was not found.
', status_code=404 + ) From 2f36a6640445f07b11c325e00fedfb4753f72a14 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 20:45:24 -0600 Subject: [PATCH 133/156] Fix missing Django template coverage in pytest --- config/settings/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config/settings/test.py b/config/settings/test.py index d7142c8b..606cee75 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -29,6 +29,7 @@ ], ), ] +TEMPLATES[0]["OPTIONS"]["debug"] = True # noqa F405 # EMAIL # ------------------------------------------------------------------------------ From 6dd341461ab369823b2915fc5c6e46bc7a042284 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 20:46:26 -0600 Subject: [PATCH 134/156] Remove latent print --- scram/route_manager/tests/test_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scram/route_manager/tests/test_views.py b/scram/route_manager/tests/test_views.py index ae46a5fa..1433135a 100644 --- a/scram/route_manager/tests/test_views.py +++ b/scram/route_manager/tests/test_views.py @@ -41,7 +41,6 @@ def test_homepage_logout_links_missing(self): response = self.client.post(reverse(settings.LOGOUT_URL), follow=True) self.assertEqual(response.status_code, 200) response = self.client.get(reverse("route_manager:home")) - print(response.content) self.assertNotContains(response, b"An admin user was created for you.") self.assertNotContains(response, b'type="submit">Logout') From 23700233ed24726595ebdaa4c15866d1558c6e02 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 23 Nov 2024 21:06:26 -0600 Subject: [PATCH 135/156] Remove temp file --- docs/.#summary.md | 1 - 1 file changed, 1 deletion(-) delete mode 120000 docs/.#summary.md diff --git a/docs/.#summary.md b/docs/.#summary.md deleted file mode 120000 index 51b0ff94..00000000 --- a/docs/.#summary.md +++ /dev/null @@ -1 +0,0 @@ -vlad@DESKTOP.855 \ No newline at end of file From 80835d4a8010c02cb51d541b1caac4808d9b08f0 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sun, 24 Nov 2024 15:25:25 -0600 Subject: [PATCH 136/156] Remove old pytest config and move to pyproject --- pytest.ini | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index c2b3a233..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = --ds=config.settings.test --reuse-db -python_files = tests.py test_*.py From ee5f0c5073d5dbf3beb3f740c9a8d69ac716581b Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 25 Nov 2024 13:26:55 -0600 Subject: [PATCH 137/156] Move docs container to mkdocs (#94) --- Makefile | 10 ++++++++ compose.override.local.yml | 8 +++--- compose/local/docs/Dockerfile | 14 +++++------ compose/local/docs/start | 8 ------ docs/Makefile | 20 --------------- docs/make.bat | 46 ----------------------------------- 6 files changed, 19 insertions(+), 87 deletions(-) delete mode 100644 compose/local/docs/start delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat diff --git a/Makefile b/Makefile index 26b6f3a0..8c3fac1f 100644 --- a/Makefile +++ b/Makefile @@ -138,3 +138,13 @@ tail-log: compose.override.yml .Phony: type-check type-check: compose.override.yml @docker compose run --rm django mypy scram + +## docs-build: build the documentation +.Phony: docs-build +docs-build: + @docker compose run --rm docs mkdocs build + +## docs-serve: build and run a server with the documentation +.Phony: docs-serve +docs-serve: + @docker compose run --rm docs mkdocs serve -a 0.0.0.0:8888 diff --git a/compose.override.local.yml b/compose.override.local.yml index c2075761..9332114a 100644 --- a/compose.override.local.yml +++ b/compose.override.local.yml @@ -44,12 +44,10 @@ services: networks: default: {} volumes: - - $CI_PROJECT_DIR/docs:/docs:z - - $CI_PROJECT_DIR/config:/app/config:z - - $CI_PROJECT_DIR/scram:/app/scram:z + - $CI_PROJECT_DIR:/app:z ports: - - "7000" - command: /start-docs + - "${DOCS_PORT:-8888}" + command: "mkdocs serve -a 0.0.0.0:${DOCS_PORT:-8888}" redis: ports: diff --git a/compose/local/docs/Dockerfile b/compose/local/docs/Dockerfile index 1aedf840..274d4829 100644 --- a/compose/local/docs/Dockerfile +++ b/compose/local/docs/Dockerfile @@ -20,13 +20,11 @@ RUN apt-get update \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && rm -rf /var/lib/apt/lists/* -# Requirements are installed here to ensure they will be cached. -COPY ./requirements /requirements -# All imports needed for autodoc. -RUN pip install -r /requirements/local.txt -r /requirements/production.txt -COPY ./compose/local/docs/start /start-docs -RUN sed -i 's/\r$//g' /start-docs -RUN chmod +x /start-docs +# Only re-run the pip install if these files have changed +COPY requirements/base.txt requirements/local.txt requirements/production.txt /app/requirements/ +RUN pip install -r /app/requirements/local.txt -r /app/requirements/production.txt -WORKDIR /docs +COPY . /app/ + +WORKDIR /app diff --git a/compose/local/docs/start b/compose/local/docs/start deleted file mode 100644 index c562c13a..00000000 --- a/compose/local/docs/start +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - -make apidocs -make livehtml diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index efd77582..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# You can set these variables from the command line, and also -# from the environment for the first two. -DOCBUILD ?= mkdocs -SOURCEDIR = . -APP = /app - -.PHONY: help livehtml Makefile - -# Put it first so that "make" without argument is like "make help". -help: - @$(DOCBUILD) -h "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -# Build, watch and serve docs with live reload -livehtml: - @$(DOCBUILD) serve -w $(SOURCEDIR) - -# Catch-all target: route all unknown targets to build -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(DOCBUILD) build diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index a22d92b7..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,46 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -c . -) -set SOURCEDIR=_source -set BUILDDIR=_build -set APP=..\scram - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.Install sphinx-autobuild for live serving. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -b %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:livehtml -sphinx-autobuild -b html --open-browser -p 7000 --watch %APP% -c . %SOURCEDIR% %BUILDDIR%/html -GOTO :EOF - -:apidocs -sphinx-apidoc -o %SOURCEDIR%/api %APP% -GOTO :EOF - -:help -%SPHINXBUILD% -b help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd From c41cdcb1c90308967414a9efe12c7b98a7e4cdbe Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 25 Nov 2024 18:16:49 -0600 Subject: [PATCH 138/156] Fix static files warning (#92) Fixes pytest warning: `No directory at: /home/runner/work/SCRAM/SCRAM/staticfiles/` --------- Co-authored-by: Chris Cummings --- .github/workflows/behave.yml | 15 +++++++++++++++ compose/local/django/start | 1 + 2 files changed, 16 insertions(+) diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml index ec12d633..30d08b27 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave.yml @@ -45,6 +45,9 @@ jobs: sudo apt-get update sudo apt-get install -y docker-compose make + - name: Check Docker state (pre-build) + run: docker ps + - name: Build Docker images run: make build @@ -54,12 +57,19 @@ jobs: - name: Run Application run: make run + - name: Check Docker state (pre-test) + run: docker ps + - name: Run pytest + behave with Coverage env: POSTGRES_USER: scram POSTGRES_DB: test_scram run: make coverage.xml + - name: Dump docker logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + - name: Upload Coverage to Coveralls uses: coverallsapp/github-action@v2 @@ -74,6 +84,11 @@ jobs: with: minimum_coverage: '50' + + - name: Check Docker state (post-test) + if: always() + run: docker ps + - name: Stop Services if: always() run: make stop diff --git a/compose/local/django/start b/compose/local/django/start index 2d910375..95f91dff 100644 --- a/compose/local/django/start +++ b/compose/local/django/start @@ -4,5 +4,6 @@ set -o errexit set -o pipefail set -o nounset +mkdir -p /app/staticfiles python manage.py migrate uvicorn config.asgi:application --host 0.0.0.0 --reload From 7e9c7cd18bb88a24022e5e8140a9f34ca968a972 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 25 Nov 2024 18:17:14 -0600 Subject: [PATCH 139/156] Annotate warnings as such in GitHub Actions for pytest (#91) --- .github/workflows/future_pytest.yml | 3 ++- .github/workflows/pytest.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/future_pytest.yml b/.github/workflows/future_pytest.yml index 58e1803b..c20d365c 100644 --- a/.github/workflows/future_pytest.yml +++ b/.github/workflows/future_pytest.yml @@ -55,7 +55,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements/local.txt - pip install pytest-github-actions-annotate-failures + # https://github.com/pytest-dev/pytest-github-actions-annotate-failures/pull/68 isn't yet in a release + pip install git+https://github.com/pytest-dev/pytest-github-actions-annotate-failures.git@6e66cd895fe05cd09be8bad58f5d79110a20385f - name: Apply unapplied migrations env: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 45feca20..5f8ca68f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -55,7 +55,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements/local.txt - pip install pytest-github-actions-annotate-failures + # https://github.com/pytest-dev/pytest-github-actions-annotate-failures/pull/68 isn't yet in a release + pip install git+https://github.com/pytest-dev/pytest-github-actions-annotate-failures.git@6e66cd895fe05cd09be8bad58f5d79110a20385f - name: Apply unapplied migrations env: From 9b93a792d2373444060a35658d3f8811a3b1271c Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Sat, 7 Dec 2024 12:06:56 -0600 Subject: [PATCH 140/156] Enable Python 3.11 and 3.12 GH Action Testing (#95) A couple extra things: * Add a Behave test with Python 3.13, similar to the pytest one * Make the Python version an argument to the Dockerfiles --- .github/workflows/behave.yml | 31 +++----- .github/workflows/behave_next_python.yml | 77 +++++++++++++++++++ .github/workflows/pytest.yml | 25 +++--- ...ture_pytest.yml => pytest_next_python.yml} | 2 +- Makefile | 7 +- compose/local/django/Dockerfile | 4 +- compose/local/docs/Dockerfile | 4 +- compose/production/django/Dockerfile | 3 +- compose/production/postgres/Dockerfile | 4 +- config/settings/local.py | 2 +- requirements/local.txt | 2 +- 11 files changed, 119 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/behave_next_python.yml rename .github/workflows/{future_pytest.yml => pytest_next_python.yml} (99%) diff --git a/.github/workflows/behave.yml b/.github/workflows/behave.yml index 30d08b27..34fb68f0 100644 --- a/.github/workflows/behave.yml +++ b/.github/workflows/behave.yml @@ -4,7 +4,7 @@ name: Run behave on: push: branches: - - '**' + - "**" pull_request: branches: - main @@ -17,22 +17,10 @@ jobs: behave: name: Run Behave runs-on: ubuntu-latest - - services: - postgres: - image: postgres:latest - env: - POSTGRES_USER: scram - POSTGRES_PASSWORD: '' - POSTGRES_DB: test_scram - POSTGRES_HOST_AUTH_METHOD: trust - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U scram" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + strategy: + max-parallel: 4 + matrix: + python-version: ["3.11", "3.12"] steps: - name: Check out the code @@ -40,6 +28,7 @@ jobs: - name: Set up Docker uses: docker/setup-buildx-action@v3 + - name: Install Docker Compose run: | sudo apt-get update @@ -50,6 +39,8 @@ jobs: - name: Build Docker images run: make build + env: + PYTHON_IMAGE_VER: "${{ matrix.python-version }}" - name: Migrate Database run: make migrate @@ -71,19 +62,21 @@ jobs: uses: jwalton/gh-docker-logs@v2 - name: Upload Coverage to Coveralls + if: matrix.python-version == '3.12' uses: coverallsapp/github-action@v2 - name: Upload Coverage to GitHub + if: matrix.python-version == '3.12' uses: actions/upload-artifact@v4 with: name: coverage-report path: coverage.xml - name: Display Coverage Metrics + if: matrix.python-version == '3.12' uses: 5monkeys/cobertura-action@v14 with: - minimum_coverage: '50' - + minimum_coverage: "50" - name: Check Docker state (post-test) if: always() diff --git a/.github/workflows/behave_next_python.yml b/.github/workflows/behave_next_python.yml new file mode 100644 index 00000000..1c4b1a3c --- /dev/null +++ b/.github/workflows/behave_next_python.yml @@ -0,0 +1,77 @@ +--- +name: Run behave with unsupported Python versions + +on: + push: + branches: + - '**' + pull_request: + branches: + - main + - develop + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + behave_next_python: + name: Run Behave + runs-on: ubuntu-latest + strategy: + max-parallel: 4 + matrix: + python-version: ['3.13'] + + steps: + - name: Check out the code + uses: actions/checkout@v4 + + - name: Set up Docker + uses: docker/setup-buildx-action@v3 + + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose make + + - name: Check Docker state (pre-build) + run: docker ps + + - name: Build Docker images + run: make build + env: + PYTHON_IMAGE_VER: "${{ matrix.python-version }}" + + - name: Migrate Database + run: | + make migrate || echo "::warning:: migrate failed on future Python version ${{ matrix.python-version }}." + + - name: Run Application + run: | + make run || echo "::warning:: run failed on future Python version ${{ matrix.python-version }}." + + - name: Check Docker state (pre-test) + run: docker ps + + - name: Run pytest + behave with Coverage + env: + POSTGRES_USER: scram + POSTGRES_DB: test_scram + run: | + make coverage.xml || echo "::warning:: pytest + behave failed on future Python version ${{ matrix.python-version }}." + + - name: Dump docker logs on failure + if: failure() + uses: jwalton/gh-docker-logs@v2 + + - name: Check Docker state (post-test) + if: always() + run: docker ps + + - name: Stop Services + if: always() + run: make stop + + - name: Clean Up + if: always() + run: make clean diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5f8ca68f..5d4004d2 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,7 +20,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ['3.12'] + python-version: ['3.11', '3.12'] services: postgres: @@ -28,7 +28,7 @@ jobs: env: POSTGRES_USER: scram POSTGRES_PASSWORD: '' - POSTGRES_DB: test_scram + POSTGRES_DB: test_scram_${{ matrix.python-version }} POSTGRES_HOST_AUTH_METHOD: trust ports: - 5432:5432 @@ -58,30 +58,25 @@ jobs: # https://github.com/pytest-dev/pytest-github-actions-annotate-failures/pull/68 isn't yet in a release pip install git+https://github.com/pytest-dev/pytest-github-actions-annotate-failures.git@6e66cd895fe05cd09be8bad58f5d79110a20385f - - name: Apply unapplied migrations + - name: Apply migrations env: - DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}" run: | - python manage.py makemigrations --noinput || true - UNAPPLIED_MIGRATIONS=$(python manage.py showmigrations --plan | grep '\[ \]' | awk '{print $2}') - if [ -n "$UNAPPLIED_MIGRATIONS" ]; then - for migration in $UNAPPLIED_MIGRATIONS; do - python manage.py migrate $migration --fake-initial --noinput - done - else - echo "No unapplied migrations." - fi + python manage.py makemigrations --noinput + python manage.py migrate --noinput - name: Check for duplicate migrations + env: + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}" run: | if python manage.py makemigrations --dry-run | grep "No changes detected"; then echo "No duplicate migrations detected." else - echo "Warning: Potential duplicate migrations detected. Please review." + echo "::warning:: Potential duplicate migrations detected. Please review." fi - name: Run Pytest env: - DATABASE_URL: "postgres://scram:@localhost:5432/test_scram" + DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}" REDIS_HOST: "localhost" run: pytest diff --git a/.github/workflows/future_pytest.yml b/.github/workflows/pytest_next_python.yml similarity index 99% rename from .github/workflows/future_pytest.yml rename to .github/workflows/pytest_next_python.yml index c20d365c..dbc9d4e1 100644 --- a/.github/workflows/future_pytest.yml +++ b/.github/workflows/pytest_next_python.yml @@ -14,7 +14,7 @@ on: workflow_dispatch: jobs: - pytest: + pytest_next_python: name: Run Pytest runs-on: ubuntu-latest strategy: diff --git a/Makefile b/Makefile index 8c3fac1f..347eb58c 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,7 @@ +# It'd be nice to keep these in sync with the defaults of the Dockerfiles +PYTHON_IMAGE_VER ?= 3.12 +POSTGRES_IMAGE_VER ?= 12.3 + .DEFAULT_GOAL := help ## toggle-prod: configure make to use the production stack @@ -37,7 +41,8 @@ behave-translator: compose.override.yml ## build: rebuilds all your containers or a single one if CONTAINER is specified .Phony: build build: compose.override.yml - @docker compose up -d --no-deps --build $(CONTAINER) + @docker compose build --build-arg PYTHON_IMAGE_VER=$(PYTHON_IMAGE_VER) --build-arg POSTGRES_IMAGE_VER=$(POSTGRES_IMAGE_VER) $(CONTAINER) + @docker compose up -d --no-deps $(CONTAINER) @docker compose restart $(CONTAINER) ## coverage.xml: generate coverage from test runs diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 12ea411b..aaba2ef9 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.12-slim-bookworm +ARG PYTHON_IMAGE_VER=3.12 + +FROM python:${PYTHON_IMAGE_VER}-slim-bookworm ENV PIP_ROOT_USER_ACTION ignore ENV PYTHONUNBUFFERED 1 diff --git a/compose/local/docs/Dockerfile b/compose/local/docs/Dockerfile index 274d4829..49965c75 100644 --- a/compose/local/docs/Dockerfile +++ b/compose/local/docs/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.12-slim-bookworm +ARG PYTHON_IMAGE_VER=3.12 + +FROM python:${PYTHON_IMAGE_VER}-slim-bookworm ENV PIP_ROOT_USER_ACTION ignore ENV PYTHONUNBUFFERED 1 diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index cbb5aeb5..84748126 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -1,5 +1,6 @@ +ARG PYTHON_IMAGE_VER=3.12 -FROM python:3.12-slim-bookwork +FROM python:${PYTHON_IMAGE_VER}-slim-bookworm ENV PYTHONUNBUFFERED 1 diff --git a/compose/production/postgres/Dockerfile b/compose/production/postgres/Dockerfile index c4160f1e..88f86a97 100644 --- a/compose/production/postgres/Dockerfile +++ b/compose/production/postgres/Dockerfile @@ -1,4 +1,6 @@ -FROM postgres:12.3 +ARG POSTGRES_IMAGE_VER=12.3 + +FROM postgres:${POSTGRES_IMAGE_VER} COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance RUN chmod +x /usr/local/bin/maintenance/* diff --git a/config/settings/local.py b/config/settings/local.py index de20d800..56029930 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -51,7 +51,7 @@ } # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips INTERNAL_IPS = ["127.0.0.1", "10.0.2.2"] -if env("USE_DOCKER") == "yes": +if env("USE_DOCKER", default="no") == "yes": import socket hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) diff --git a/requirements/local.txt b/requirements/local.txt index 2712cf7e..134a2213 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -2,7 +2,7 @@ Werkzeug[watchdog]==2.0.3 # https://github.com/pallets/werkzeug ipdb==0.13.9 # https://github.com/gotcha/ipdb -psycopg2-binary==2.9.3 # https://github.com/psycopg/psycopg2 +psycopg2-binary==2.9.10 # https://github.com/psycopg/psycopg2 watchgod==0.8.2 # https://github.com/samuelcolvin/watchgod # Testing From 0667c54b3a95eda45aedde5de722ff51789f0a01 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Tue, 10 Dec 2024 10:29:10 -0600 Subject: [PATCH 141/156] feat(db): Enable running postgres locally in prod-mode (#93) This lets you run prod with a local postgres instance. We prolly need to document the vault stuff, but I think that's better suited for #88 to handle. I *think* this MR is ready to rock, it's deployed and working on `scram-pentest.ocean.cu-es.net` presently. --- compose.override.production.yml | 3 ++- compose/production/django/Dockerfile | 2 ++ config/settings/production.py | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compose.override.production.yml b/compose.override.production.yml index 58f47d72..de54d36a 100644 --- a/compose.override.production.yml +++ b/compose.override.production.yml @@ -26,8 +26,9 @@ services: - production_postgres_data_backups:/backups:z env_file: - ./.envs/.production/.postgres + - /etc/vault.d/secrets/kv_root_security.env deploy: - replicas: 0 + replicas: ${POSTGRES_ENABLED:-0} nginx: image: nginx:1.19 diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index 84748126..f9e1407e 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -1,7 +1,9 @@ ARG PYTHON_IMAGE_VER=3.12 + FROM python:${PYTHON_IMAGE_VER}-slim-bookworm + ENV PYTHONUNBUFFERED 1 RUN apt-get update \ diff --git a/config/settings/production.py b/config/settings/production.py index cdb213f0..4af0de77 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -13,9 +13,10 @@ DATABASES["default"] = env.db("DATABASE_URL") # noqa F405 DATABASES["default"]["ATOMIC_REQUESTS"] = True # noqa F405 DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) # noqa F405 -if env("POSTGRES_SSL"): +if env.bool("POSTGRES_SSL", default=True): DATABASES["default"]["OPTIONS"] = {"sslmode": "require"} # noqa F405 - +else: + DATABASES["default"]["OPTIONS"] = {"sslmode": "disable"} # noqa F405 # CACHES # ------------------------------------------------------------------------------ CACHES = { From 6810e7497e8d22e4c56c1faf498c607b1c3782d2 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Mon, 23 Dec 2024 13:12:36 -0600 Subject: [PATCH 142/156] chore(release): merge hotfix back to develop (#104) release(1.1.1) From d2ed5b3acbee92641ff79e4526ed1b1ec976818b Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 24 Dec 2024 08:52:55 -0600 Subject: [PATCH 143/156] Add missing migration and ensure we fail if we're missing future ones (#107) --- .github/workflows/pytest.yml | 9 +--- ..._alter_entry_expiration_reason_and_more.py | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 scram/route_manager/migrations/0030_alter_entry_comment_alter_entry_expiration_reason_and_more.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 5d4004d2..6808d154 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -65,15 +65,10 @@ jobs: python manage.py makemigrations --noinput python manage.py migrate --noinput - - name: Check for duplicate migrations + - name: Check for missing migrations env: DATABASE_URL: "postgres://scram:@localhost:5432/test_scram_${{ matrix.python-version }}" - run: | - if python manage.py makemigrations --dry-run | grep "No changes detected"; then - echo "No duplicate migrations detected." - else - echo "::warning:: Potential duplicate migrations detected. Please review." - fi + run: python manage.py makemigrations --check - name: Run Pytest env: diff --git a/scram/route_manager/migrations/0030_alter_entry_comment_alter_entry_expiration_reason_and_more.py b/scram/route_manager/migrations/0030_alter_entry_comment_alter_entry_expiration_reason_and_more.py new file mode 100644 index 00000000..b16ae4ae --- /dev/null +++ b/scram/route_manager/migrations/0030_alter_entry_comment_alter_entry_expiration_reason_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.17 on 2024-12-24 13:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("route_manager", "0029_alter_websocketmessage_msg_data_route_field"), + ] + + operations = [ + migrations.AlterField( + model_name="entry", + name="comment", + field=models.TextField(blank=True, default=""), + ), + migrations.AlterField( + model_name="entry", + name="expiration_reason", + field=models.CharField( + blank=True, + default="", + help_text="Optional reason for the expiration", + max_length=200, + ), + ), + migrations.AlterField( + model_name="historicalentry", + name="comment", + field=models.TextField(blank=True, default=""), + ), + migrations.AlterField( + model_name="historicalentry", + name="expiration_reason", + field=models.CharField( + blank=True, + default="", + help_text="Optional reason for the expiration", + max_length=200, + ), + ), + ] From b39949b1d9230dacc433687e4e929cf15e00ab42 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 24 Dec 2024 15:21:50 -0600 Subject: [PATCH 144/156] Add performance tests for number of queries and response time. (#109) --- scram/route_manager/tests/test_performance.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 scram/route_manager/tests/test_performance.py diff --git a/scram/route_manager/tests/test_performance.py b/scram/route_manager/tests/test_performance.py new file mode 100644 index 00000000..7d507865 --- /dev/null +++ b/scram/route_manager/tests/test_performance.py @@ -0,0 +1,77 @@ +"""Tests for performance (load time, DB queries, etc.).""" + +import time + +from django.test import TestCase +from django.urls import reverse +from faker import Faker +from faker.providers import internet + +from scram.route_manager.models import ActionType, Entry, Route + + +class TestViewNumQueries(TestCase): + """Viewing an entry should only require one query.""" + + NUM_ENTRIES = 100_000 + + def setUp(self): + """Set up the test environment.""" + self.fake = Faker() + self.fake.add_provider(internet) + + # Query the homepage once to setup the user + self.client.get(reverse("route_manager:home")) + + self.atype = ActionType.objects.create(name="Block") + routes = [Route(route=self.fake.unique.ipv4_public()) for x in range(self.NUM_ENTRIES)] + created_routes = Route.objects.bulk_create(routes) + entries = [Entry(route=route, actiontype=self.atype) for route in created_routes] + Entry.objects.bulk_create(entries) + + def test_home_page(self): + """Home page requires 30 queries. + + 1. create transaction + 2. lookup session + 3. lookup user + 4. filter available actiontypes + 5. count entries with actiontype=1 + 6. count entries with actiontype=33 + 7. count by user + 8. first page for actiontype=1 + 9. first page for actiontype=33 + [ for each of the 10 elements on the first page: ] + n. actiontype info + n+1. entry info + [ endfor ] + # 9 + 2*10 = 29 + 30. close transaction + + """ + with self.assertNumQueries(30): + start = time.time() + self.client.get(reverse("route_manager:home")) + time_taken = time.time() - start + self.assertLess(time_taken, 1, "Home page took longer than 1 second") + + def test_admin_entry_page(self): + """Admin entry list page requires 207 queries. + + 1. create transaction + 2. lookup session + 3. lookup user + 4. count entries + 5. count entries + 6. get first 100 entries + 7. release transaction + [ for each of the 100 elements on the first page: ] + n. actiontype info + n+1. entry info + [ endfor ] + """ + with self.assertNumQueries(207): + start = time.time() + self.client.get(reverse("admin:route_manager_entry_changelist")) + time_taken = time.time() - start + self.assertLess(time_taken, 1, "Admin entry list page took longer than 1 seconds") From cf24e23e7a285e9e220aa9a6aba5e4b03b388804 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Wed, 25 Dec 2024 08:32:43 -0600 Subject: [PATCH 145/156] Hotfix: build the docs (#105) --- .github/workflows/docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 223f58d5..aacca837 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -41,6 +41,9 @@ jobs: - name: "[pytest]: before" run: "./.ci-scripts/pytest_before.sh" + - name: Build docs + run: make docs-build + - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: From d71aa6f597fca2cf5333069edd44eacf63ef06c6 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Wed, 25 Dec 2024 22:08:15 -0600 Subject: [PATCH 146/156] feat(entry admin): Improve the UX on the entry admin panel (#108) Closes #99 Closes #98 I had to branch off of this to make some other changes as well unfortunately just to get a working copy of the stack. Our expiration was not TZ aware which broke any add (from WUI or admin). Also had to fix the translatorConsumer because websockets were broken and spewing errors constantly. --- config/consumers.py | 3 +- pyproject.toml | 3 ++ scram/route_manager/admin.py | 37 +++++++++++++++- scram/route_manager/api/serializers.py | 2 +- .../route_manager/authentication_backends.py | 4 +- .../0031_alter_entry_expiration_and_more.py | 35 +++++++++++++++ scram/route_manager/models.py | 19 ++++---- scram/route_manager/tests/test_admin.py | 44 +++++++++++++++++++ scram/route_manager/tests/test_performance.py | 5 ++- 9 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 scram/route_manager/migrations/0031_alter_entry_expiration_and_more.py create mode 100644 scram/route_manager/tests/test_admin.py diff --git a/config/consumers.py b/config/consumers.py index cd163bb7..acaf11c9 100644 --- a/config/consumers.py +++ b/config/consumers.py @@ -1,7 +1,6 @@ """Define logic for the WebSocket consumers.""" import logging -from functools import partial from asgiref.sync import sync_to_async from channels.generic.websocket import AsyncJsonWebsocketConsumer @@ -36,7 +35,7 @@ async def connect(self): for route in routes: for element in elements: - msg = await sync_to_async(partial(element.websocketmessage)) + msg = await sync_to_async(lambda e: e.websocketmessage)(element) msg.msg_data[msg.msg_data_route_field] = str(route) await self.send_json({"type": msg.msg_type, "message": msg.msg_data}) diff --git a/pyproject.toml b/pyproject.toml index 5c332104..9a3043ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,9 @@ max-complexity = 7 # our current code adheres to this without too much effort "test.py" = [ "S105", # hardcoded password as argument ] +"scram/route_manager/**" = [ + "DOC201", # documenting return values +] "scram/users/**" = [ "DOC201", # documenting return values "FBT001", # minimal issue; don't need to mess with in the User app diff --git a/scram/route_manager/admin.py b/scram/route_manager/admin.py index 172dd6c7..9387b57b 100644 --- a/scram/route_manager/admin.py +++ b/scram/route_manager/admin.py @@ -6,6 +6,31 @@ from .models import ActionType, Client, Entry, IgnoreEntry, Route, WebSocketMessage, WebSocketSequenceElement +class WhoFilter(admin.SimpleListFilter): + """Only display users who have added entries in the list_filter.""" + + title = "By Username" + parameter_name = "who" + + # ruff: noqa: PLR6301 + def lookups(self, request, model_admin): + """Return list of users who have added entries.""" + users_with_entries = Entry.objects.values("who").distinct() + + # If no users have entries, return an empty list so they don't show in filter + if not users_with_entries: + return [] + + # Return a list of users who have made entries + return [(user["who"], user["who"]) for user in users_with_entries] + + def queryset(self, request, queryset): + """Queryset for users.""" + if self.value(): + return queryset.filter(who=self.value()) + return queryset + + @admin.register(ActionType) class ActionTypeAdmin(SimpleHistoryAdmin): """Configure the ActionType and how it shows up in the Admin site.""" @@ -14,7 +39,17 @@ class ActionTypeAdmin(SimpleHistoryAdmin): list_display = ("name", "available") -admin.site.register(Entry, SimpleHistoryAdmin) +@admin.register(Entry) +class EntryAdmin(SimpleHistoryAdmin): + """Configure how Entries show up in the Admin site.""" + + list_filter = [ + "is_active", + WhoFilter, + ] + search_fields = ["route", "comment"] + + admin.site.register(IgnoreEntry, SimpleHistoryAdmin) admin.site.register(Route) admin.site.register(Client) diff --git a/scram/route_manager/api/serializers.py b/scram/route_manager/api/serializers.py index f1bc1907..62ade3f7 100644 --- a/scram/route_manager/api/serializers.py +++ b/scram/route_manager/api/serializers.py @@ -86,7 +86,7 @@ def get_comment(obj): @staticmethod def create(validated_data): - """Implement custom logic and validates creating a new route.""" # noqa: DOC201 + """Implement custom logic and validates creating a new route.""" valid_route = validated_data.pop("route") actiontype = validated_data.pop("actiontype") comment = validated_data.pop("comment") diff --git a/scram/route_manager/authentication_backends.py b/scram/route_manager/authentication_backends.py index 1831d788..0c7c2a4b 100644 --- a/scram/route_manager/authentication_backends.py +++ b/scram/route_manager/authentication_backends.py @@ -38,12 +38,12 @@ def update_groups(user, claims): user.save() def create_user(self, claims): - """Wrap the superclass's user creation.""" # noqa: DOC201 + """Wrap the superclass's user creation.""" user = super().create_user(claims) return self.update_user(user, claims) def update_user(self, user, claims): - """Determine the user name from the claims and update said user's groups.""" # noqa: DOC201 + """Determine the user name from the claims and update said user's groups.""" user.name = claims.get("given_name", "") + " " + claims.get("family_name", "") user.username = claims.get("preferred_username", "") if claims.get("groups", False): diff --git a/scram/route_manager/migrations/0031_alter_entry_expiration_and_more.py b/scram/route_manager/migrations/0031_alter_entry_expiration_and_more.py new file mode 100644 index 00000000..4306e135 --- /dev/null +++ b/scram/route_manager/migrations/0031_alter_entry_expiration_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.17 on 2024-12-24 16:02 + +import datetime +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "route_manager", + "0030_alter_entry_comment_alter_entry_expiration_reason_and_more", + ), + ] + + operations = [ + migrations.AlterField( + model_name="entry", + name="expiration", + field=models.DateTimeField( + default=datetime.datetime( + 9999, 12, 31, 0, 0, tzinfo=datetime.timezone.utc + ) + ), + ), + migrations.AlterField( + model_name="historicalentry", + name="expiration", + field=models.DateTimeField( + default=datetime.datetime( + 9999, 12, 31, 0, 0, tzinfo=datetime.timezone.utc + ) + ), + ), + ] diff --git a/scram/route_manager/models.py b/scram/route_manager/models.py index aa72f7ba..b120fe6e 100644 --- a/scram/route_manager/models.py +++ b/scram/route_manager/models.py @@ -1,5 +1,6 @@ """Define the models used in the route_manager app.""" +import datetime import logging import uuid as uuid_lib @@ -20,12 +21,12 @@ class Route(models.Model): uuid = models.UUIDField(db_index=True, default=uuid_lib.uuid4, editable=False) def __str__(self): - """Don't display the UUID, only the route.""" # noqa: DOC201 + """Don't display the UUID, only the route.""" return str(self.route) @staticmethod def get_absolute_url(): - """Ensure we use UUID on the API side instead.""" # noqa: DOC201 + """Ensure we use UUID on the API side instead.""" return reverse("") @@ -37,7 +38,7 @@ class ActionType(models.Model): history = HistoricalRecords() def __str__(self): - """Display clearly whether the action is currently available.""" # noqa: DOC201 + """Display clearly whether the action is currently available.""" if not self.available: return f"{self.name} (Inactive)" return self.name @@ -55,7 +56,7 @@ class WebSocketMessage(models.Model): ) def __str__(self): - """Display clearly what the fields are used for.""" # noqa: DOC201 + """Display clearly what the fields are used for.""" return f"{self.msg_type}: {self.msg_data} with the route in key {self.msg_data_route_field}" @@ -79,7 +80,7 @@ class WebSocketSequenceElement(models.Model): action_type = models.ForeignKey("ActionType", on_delete=models.CASCADE) def __str__(self): - """Summarize the fields into something short and readable.""" # noqa: DOC201 + """Summarize the fields into something short and readable.""" return ( f"{self.websocketmessage} as order={self.order_num} for " f"{self.verb} actions on actiontype={self.action_type}" @@ -97,7 +98,7 @@ class Entry(models.Model): history = HistoricalRecords() when = models.DateTimeField(auto_now_add=True) who = models.CharField("Username", default="Unknown", max_length=30) - expiration = models.DateTimeField(default="9999-12-31 00:00") + expiration = models.DateTimeField(default=datetime.datetime(9999, 12, 31, 0, 0, tzinfo=datetime.UTC)) expiration_reason = models.CharField( help_text="Optional reason for the expiration", max_length=200, @@ -112,7 +113,7 @@ class Meta: verbose_name_plural = "Entries" def __str__(self): - """Summarize the most important fields to something easily readable.""" # noqa: DOC201 + """Summarize the most important fields to something easily readable.""" desc = f"{self.route} ({self.actiontype})" if not self.is_active: desc += " (inactive)" @@ -160,7 +161,7 @@ class Meta: verbose_name_plural = "Ignored Entries" def __str__(self): - """Only display the route.""" # noqa: DOC201 + """Only display the route.""" return str(self.route) @@ -174,7 +175,7 @@ class Client(models.Model): authorized_actiontypes = models.ManyToManyField(ActionType) def __str__(self): - """Only display the hostname.""" # noqa: DOC201 + """Only display the hostname.""" return str(self.hostname) diff --git a/scram/route_manager/tests/test_admin.py b/scram/route_manager/tests/test_admin.py new file mode 100644 index 00000000..4cf0a4d3 --- /dev/null +++ b/scram/route_manager/tests/test_admin.py @@ -0,0 +1,44 @@ +"""Test the WhoFilter functionality of our admin site.""" + +from unittest.mock import MagicMock + +from django.test import TestCase + +from scram.route_manager.admin import EntryAdmin, WhoFilter +from scram.route_manager.models import ActionType, Entry, Route + + +class WhoFilterTest(TestCase): + """Test that the WhoFilter only shows users who have made entries.""" + + def setUp(self): + """Set up the test environment.""" + self.atype = ActionType.objects.create(name="Block") + route1 = Route.objects.create(route="192.168.1.1") + route2 = Route.objects.create(route="192.168.1.2") + + self.entry1 = Entry.objects.create(route=route1, actiontype=self.atype, who="admin") + self.entry2 = Entry.objects.create(route=route2, actiontype=self.atype, who="user1") + + def test_who_filter_lookups(self): + """Test that the WhoFilter returns the correct users who have made entries.""" + who_filter = WhoFilter(request=None, params={}, model=Entry, model_admin=EntryAdmin) + + mock_request = MagicMock() + mock_model_admin = MagicMock(spec=EntryAdmin) + + result = who_filter.lookups(mock_request, mock_model_admin) + + self.assertIn(("admin", "admin"), result) + self.assertIn(("user1", "user1"), result) + self.assertEqual(len(result), 2) # Only two users should be present + + def test_who_filter_queryset_with_value(self): + """Test that the queryset is filtered correctly when a user is selected.""" + who_filter = WhoFilter(request=None, params={"who": "admin"}, model=Entry, model_admin=EntryAdmin) + + queryset = Entry.objects.all() + filtered_queryset = who_filter.queryset(None, queryset) + + self.assertEqual(filtered_queryset.count(), 1) + self.assertEqual(filtered_queryset.first(), self.entry1) diff --git a/scram/route_manager/tests/test_performance.py b/scram/route_manager/tests/test_performance.py index 7d507865..348c2d04 100644 --- a/scram/route_manager/tests/test_performance.py +++ b/scram/route_manager/tests/test_performance.py @@ -56,11 +56,12 @@ def test_home_page(self): self.assertLess(time_taken, 1, "Home page took longer than 1 second") def test_admin_entry_page(self): - """Admin entry list page requires 207 queries. + """Admin entry list page requires 208 queries. 1. create transaction 2. lookup session 3. lookup user + 4. lookup distinct users for our WhoFilter 4. count entries 5. count entries 6. get first 100 entries @@ -70,7 +71,7 @@ def test_admin_entry_page(self): n+1. entry info [ endfor ] """ - with self.assertNumQueries(207): + with self.assertNumQueries(208): start = time.time() self.client.get(reverse("admin:route_manager_entry_changelist")) time_taken = time.time() - start From 12fa5614b2528177a17937eb51071b903171b8db Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Thu, 26 Dec 2024 08:45:22 -0600 Subject: [PATCH 147/156] Use select_related for admin.Entry (#111) Reduce the number of DB queries needed by 26x. --- scram/route_manager/admin.py | 2 ++ scram/route_manager/tests/test_performance.py | 9 +++------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scram/route_manager/admin.py b/scram/route_manager/admin.py index 9387b57b..00d267b3 100644 --- a/scram/route_manager/admin.py +++ b/scram/route_manager/admin.py @@ -43,6 +43,8 @@ class ActionTypeAdmin(SimpleHistoryAdmin): class EntryAdmin(SimpleHistoryAdmin): """Configure how Entries show up in the Admin site.""" + list_select_related = True + list_filter = [ "is_active", WhoFilter, diff --git a/scram/route_manager/tests/test_performance.py b/scram/route_manager/tests/test_performance.py index 348c2d04..f71bce92 100644 --- a/scram/route_manager/tests/test_performance.py +++ b/scram/route_manager/tests/test_performance.py @@ -65,13 +65,10 @@ def test_admin_entry_page(self): 4. count entries 5. count entries 6. get first 100 entries - 7. release transaction - [ for each of the 100 elements on the first page: ] - n. actiontype info - n+1. entry info - [ endfor ] + 7. query entries (a single query, with select_related) + 8. release transaction """ - with self.assertNumQueries(208): + with self.assertNumQueries(8): start = time.time() self.client.get(reverse("admin:route_manager_entry_changelist")) time_taken = time.time() - start From 753177de69391a7f901b16fca5bf62ecd28fb1f4 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Fri, 3 Jan 2025 11:58:57 -0600 Subject: [PATCH 148/156] Modify expiration settings for channels (#114) --- config/settings/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/settings/base.py b/config/settings/base.py index be90d71e..86fb4b21 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -256,6 +256,8 @@ "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", "CONFIG": { + "expiry": 86400 * 7, # expire messages after a week (default 60s) + "group_expiry": 86400 * 365 * 10, # effectively disable removing from a group (default 1d) "hosts": [(os.environ.get("REDIS_HOST", "redis"), 6379)], }, }, From 8bd86d26fc628be00603c012247b53e022cf6500 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Fri, 3 Jan 2025 12:02:36 -0600 Subject: [PATCH 149/156] A couple more query optimizations and performance checks (#112) --- scram/route_manager/tests/test_performance.py | 53 ++++++++++++++----- scram/route_manager/views.py | 14 ++--- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/scram/route_manager/tests/test_performance.py b/scram/route_manager/tests/test_performance.py index f71bce92..e12eceed 100644 --- a/scram/route_manager/tests/test_performance.py +++ b/scram/route_manager/tests/test_performance.py @@ -23,40 +23,50 @@ def setUp(self): # Query the homepage once to setup the user self.client.get(reverse("route_manager:home")) - self.atype = ActionType.objects.create(name="Block") + self.atype, _ = ActionType.objects.get_or_create(name="block") routes = [Route(route=self.fake.unique.ipv4_public()) for x in range(self.NUM_ENTRIES)] created_routes = Route.objects.bulk_create(routes) entries = [Entry(route=route, actiontype=self.atype) for route in created_routes] Entry.objects.bulk_create(entries) def test_home_page(self): - """Home page requires 30 queries. + """Home page requires 8 queries. 1. create transaction 2. lookup session 3. lookup user 4. filter available actiontypes 5. count entries with actiontype=1 - 6. count entries with actiontype=33 - 7. count by user - 8. first page for actiontype=1 - 9. first page for actiontype=33 - [ for each of the 10 elements on the first page: ] - n. actiontype info - n+1. entry info - [ endfor ] - # 9 + 2*10 = 29 - 30. close transaction + 6. count by user + 7. first page for actiontype=1 + 8. close transaction """ - with self.assertNumQueries(30): + with self.assertNumQueries(8): start = time.time() self.client.get(reverse("route_manager:home")) time_taken = time.time() - start self.assertLess(time_taken, 1, "Home page took longer than 1 second") + def test_entry_view(self): + """Viewing an entry requires 6 queries. + + 1. create transaction savepoint + 2. lookup session + 3. lookup user + 4. get entry + 5. rollback to savepoint + 6. release transaction savepoint + + """ + with self.assertNumQueries(6): + start = time.time() + self.client.get(reverse("route_manager:detail", kwargs={"pk": 9999})) + time_taken = time.time() - start + self.assertLess(time_taken, 1, "Entry detail page took longer than 1 second") + def test_admin_entry_page(self): - """Admin entry list page requires 208 queries. + """Admin entry list page requires 8 queries. 1. create transaction 2. lookup session @@ -73,3 +83,18 @@ def test_admin_entry_page(self): self.client.get(reverse("admin:route_manager_entry_changelist")) time_taken = time.time() - start self.assertLess(time_taken, 1, "Admin entry list page took longer than 1 seconds") + + def test_process_expired(self): + """Process expired requires 5 queries. + + 1. create transaction + 2. get entries_start active entry count + 3. find and delete expired entries + 4. get entries_end active entry count + 5. release transaction + """ + with self.assertNumQueries(5): + start = time.time() + self.client.get(reverse("route_manager:process-expired")) + time_taken = time.time() - start + self.assertLess(time_taken, 1, "Process expired page took longer than 1 seconds") diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index fa8b7116..71b15440 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -27,7 +27,7 @@ def home_page(request, prefilter=None): """Return the home page, autocreating a user if none exists.""" if not prefilter: - prefilter = Entry.objects.all() + prefilter = Entry.objects.all().select_related("actiontype", "route") num_entries = settings.RECENT_LIMIT if request.user.has_perms(("route_manager.view_entry", "route_manager.add_entry")): readwrite = True @@ -130,12 +130,14 @@ def add_entry(request): def process_expired(request): """For entries with an expiration, set them to inactive if expired. Return some simple stats.""" + # This operation should be atomic, but we set ATOMIC_REQUESTS=True current_time = timezone.now() - with transaction.atomic(): - entries_start = Entry.objects.filter(is_active=True).count() - for obj in Entry.objects.filter(is_active=True, expiration__lt=current_time): - obj.delete() - entries_end = Entry.objects.filter(is_active=True).count() + entries_start = Entry.objects.filter(is_active=True).count() + + # More efficient to call objects.filter.delete, but that doesn't call the Entry.delete() method + for obj in Entry.objects.filter(is_active=True, expiration__lt=current_time): + obj.delete() + entries_end = Entry.objects.filter(is_active=True).count() return HttpResponse( json.dumps( From dbff9fc9e4b1f5e8829c842f80968daf87010875 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Mon, 6 Jan 2025 10:57:49 -0600 Subject: [PATCH 150/156] feat(logging): Log properly from docker (#101) --- compose.override.production.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compose.override.production.yml b/compose.override.production.yml index de54d36a..0583855e 100644 --- a/compose.override.production.yml +++ b/compose.override.production.yml @@ -11,6 +11,10 @@ services: build: dockerfile: ./compose/production/django/Dockerfile image: scram_production_django + logging: + driver: journald + options: + tag: scram-django env_file: - ./.envs/.production/.django - ./.envs/.production/.postgres @@ -21,6 +25,10 @@ services: test: ["CMD", "curl", "-f", "http://django:5000/process_expired/"] postgres: + logging: + driver: journald + options: + tag: scram-postgres volumes: - production_postgres_data:/var/lib/postgresql/data:Z - production_postgres_data_backups:/backups:z @@ -32,6 +40,10 @@ services: nginx: image: nginx:1.19 + logging: + driver: journald + options: + tag: scram-nginx restart: on-failure:5 depends_on: - django @@ -51,10 +63,18 @@ services: - "80:80" redis: + logging: + driver: journald + options: + tag: scram-redis volumes: - production_redis_data:/var/lib/redis:Z gobgp: + logging: + driver: journald + options: + tag: scram-gobgp volumes: - ./gobgp_config:/config:z networks: @@ -67,6 +87,10 @@ services: - "50051" translator: + logging: + driver: journald + options: + tag: scram-translator env_file: - ./.envs/.production/.translator From f9e9bfc9e8609b7da9e53f5a617b04800ba1bcf6 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Thu, 9 Jan 2025 08:38:17 -0600 Subject: [PATCH 151/156] chore(deps): bump gobgp to latest (v3.33.0) (#119) Let's move GoBGP to a recent version to reduce future suffering. --- compose.yml | 2 +- compose/local/translator/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.yml b/compose.yml index 9a08b07c..70d7dd44 100644 --- a/compose.yml +++ b/compose.yml @@ -44,7 +44,7 @@ services: test: ["CMD", "redis-cli", "--raw", "incr", "ping"] gobgp: - image: jauderho/gobgp:v2.32.0 + image: jauderho/gobgp:v3.33.0 networks: default: {} sysctls: diff --git a/compose/local/translator/Dockerfile b/compose/local/translator/Dockerfile index 21ec884a..4e0fe602 100644 --- a/compose/local/translator/Dockerfile +++ b/compose/local/translator/Dockerfile @@ -16,7 +16,7 @@ RUN pip install -r /requirements/base.txt RUN mkdir /app \ && cd /app \ - && git clone -b v2.32.0 https://github.com/osrg/gobgp.git \ + && git clone -b v3.33.0 https://github.com/osrg/gobgp.git \ && cd gobgp/api \ && python3 -m grpc_tools.protoc -I./ --python_out=/app/ --grpc_python_out=/app/ *.proto From 1d29947da135f839797b8dee075e0714a7178f93 Mon Sep 17 00:00:00 2001 From: Chris Cummings Date: Thu, 9 Jan 2025 13:44:31 -0600 Subject: [PATCH 152/156] feat(translator): type-stubs and grpc helper command (#121) Now when we generate our gRPC library, we also generate type stubs and provide a way to copy-back the auto-generated files if you want so that we can have better IDE integration (i.e. run `make copy-libs` and it will pop them into the translator directory) --- .gitignore | 3 +++ Makefile | 13 +++++++++++++ compose/local/translator/Dockerfile | 2 +- translator/requirements/base.txt | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 959510e4..650900c1 100644 --- a/.gitignore +++ b/.gitignore @@ -345,3 +345,6 @@ compose.override.yml coverage.coverage coverage.xml + +# Ignore copied-back autogenerated grpc library +translator/*pb2*.py* diff --git a/Makefile b/Makefile index 347eb58c..b40e8304 100644 --- a/Makefile +++ b/Makefile @@ -153,3 +153,16 @@ docs-build: .Phony: docs-serve docs-serve: @docker compose run --rm docs mkdocs serve -a 0.0.0.0:8888 + +## copy-libs: copy the translator autogenerated libraries into the translator directory +.Phony: copy-libs +copy-libs: + @docker compose cp translator:/app/gobgp_pb2.py translator/ + @docker compose cp translator:/app/gobgp_pb2.pyi translator/ + @docker compose cp translator:/app/gobgp_pb2_grpc.py translator/ + @docker compose cp translator:/app/attribute_pb2.py translator/ + @docker compose cp translator:/app/attribute_pb2.pyi translator/ + @docker compose cp translator:/app/attribute_pb2_grpc.py translator/ + @docker compose cp translator:/app/capability_pb2.py translator/ + @docker compose cp translator:/app/capability_pb2.pyi translator/ + @docker compose cp translator:/app/capability_pb2_grpc.py translator/ diff --git a/compose/local/translator/Dockerfile b/compose/local/translator/Dockerfile index 4e0fe602..ffa702c0 100644 --- a/compose/local/translator/Dockerfile +++ b/compose/local/translator/Dockerfile @@ -18,7 +18,7 @@ RUN mkdir /app \ && cd /app \ && git clone -b v3.33.0 https://github.com/osrg/gobgp.git \ && cd gobgp/api \ - && python3 -m grpc_tools.protoc -I./ --python_out=/app/ --grpc_python_out=/app/ *.proto + && python3 -m grpc_tools.protoc -I./ --python_out=/app/ --pyi_out=/app/ --grpc_python_out=/app/ *.proto COPY ./translator/translator.py /app COPY ./translator/gobgp.py /app diff --git a/translator/requirements/base.txt b/translator/requirements/base.txt index ae61c70a..7ffb0c44 100644 --- a/translator/requirements/base.txt +++ b/translator/requirements/base.txt @@ -1,5 +1,5 @@ aiohttp-sse-client==0.2.1 behave~=1.2.6 coverage==5.5 -grpcio-tools==1.41.0 +grpcio-tools==1.69.0 websockets==10.3 From 0d45c6bbfeb787b52f620bcc9d3dba567f0dd7a2 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Fri, 17 Jan 2025 12:38:27 -0600 Subject: [PATCH 153/156] Merge main into develop (#124) Co-authored-by: Chris Cummings Co-authored-by: Sam Oehlert From 5bc37139dec10c6c086f65e78fee3a1302244465 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 21 Jan 2025 15:22:04 -0600 Subject: [PATCH 154/156] Enable ENV var control of replicas. This allows failure testing. (#122) --- compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose.yml b/compose.yml index 70d7dd44..44c3e048 100644 --- a/compose.yml +++ b/compose.yml @@ -19,6 +19,8 @@ services: timeout: 30s start_period: 30s retries: 5 + deploy: + replicas: ${DJANGO_REPLICAS:-1} postgres: build: @@ -51,6 +53,8 @@ services: - net.ipv6.conf.all.disable_ipv6=0 healthcheck: test: ["CMD", "gobgp", "global"] + deploy: + replicas: ${GOBGP_REPLICAS:-1} translator: build: @@ -65,3 +69,5 @@ services: default: {} sysctls: - net.ipv6.conf.all.disable_ipv6=0 + deploy: + replicas: ${TRANSLATOR_REPLICAS:-1} From ed4af5622497587cdc667022ecd039ca2d76125e Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Sat, 25 Jan 2025 14:07:36 -0600 Subject: [PATCH 155/156] fix(add_form): redirect to the homepage after adding via the form on the home page (#128) Closes #113 This was an annoying UX bug where you would be on a url ending in `/add` meaning you couldn't refresh or anything since it's a POST only endpoint. This way things are still added, but now you are redirected to the actual home page. --- scram/route_manager/tests/test_authorization.py | 4 ++-- scram/route_manager/views.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scram/route_manager/tests/test_authorization.py b/scram/route_manager/tests/test_authorization.py index 9a0d4bff..596455c3 100644 --- a/scram/route_manager/tests/test_authorization.py +++ b/scram/route_manager/tests/test_authorization.py @@ -95,7 +95,7 @@ def test_authorized_add_entry(self): "uuid": "0e7e1cbd-7d73-4968-bc4b-ce3265dc2fd3", }, ) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 302) def test_unauthorized_detail_view(self): """Ensure that unauthorized users can't view the blocked IPs.""" @@ -124,7 +124,7 @@ def test_unauthorized_after_group_removal(self): self.client.force_login(test_user) response = self.client.post(reverse("route_manager:add"), {"route": "192.0.2.4/32", "actiontype": "block"}) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 302) test_user.groups.set([]) diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index 71b15440..ff282f7e 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -124,8 +124,8 @@ def add_entry(request): else: messages.add_message(request, messages.WARNING, f"Something went wrong: {res.status_code}") with transaction.atomic(): - home = home_page(request) - return home # noqa RET504 + home_page(request) + return redirect("route_manager:home") def process_expired(request): From 913e43199236690c1ed51ea1f7176b5031117fb5 Mon Sep 17 00:00:00 2001 From: Sam Oehlert Date: Mon, 3 Feb 2025 15:19:30 -0600 Subject: [PATCH 156/156] feat(active_blocks): Add a quick way to tell how many active blocks there are (#127) Closes #30 This adds a pill badge to the navbar with a quick view at active blocks. Also, it filters the homepage to only show active blocks and update the pill badge above the actiontype list on the homepage to only show the count of active blocks. The navbar is specific to blocks, but that seems fair since 1) that's the main use case right now 2) that's the only default actiontype anyways. We can always make it nicer/configurable in the future if needed. --- config/settings/base.py | 1 + scram/route_manager/context_processors.py | 15 +++++++++++++ scram/route_manager/tests/test_performance.py | 22 ++++++++++--------- scram/route_manager/views.py | 6 ++--- scram/templates/navbar.html | 6 +++++ scram/templates/route_manager/home.html | 2 +- 6 files changed, 38 insertions(+), 14 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 86fb4b21..04524490 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -186,6 +186,7 @@ "django.contrib.messages.context_processors.messages", "scram.utils.context_processors.settings_context", "scram.route_manager.context_processors.login_logout", + "scram.route_manager.context_processors.active_count", ], }, }, diff --git a/scram/route_manager/context_processors.py b/scram/route_manager/context_processors.py index 0daf06c9..b87dcf70 100644 --- a/scram/route_manager/context_processors.py +++ b/scram/route_manager/context_processors.py @@ -3,6 +3,8 @@ from django.conf import settings from django.urls import reverse +from scram.route_manager.models import Entry + def login_logout(request): """Pass through the relevant URLs from the settings. @@ -13,3 +15,16 @@ def login_logout(request): login_url = reverse(settings.LOGIN_URL) logout_url = reverse(settings.LOGOUT_URL) return {"login": login_url, "logout": logout_url} + + +def active_count(request): + """Grab the active count of blocks. + + Returns: + dict: active count of blocks + """ + if "admin" not in request.META["PATH_INFO"]: + active_block_entries = Entry.objects.filter(is_active=True).count() + total_block_entries = Entry.objects.all().count() + return {"active_block_entries": active_block_entries, "total_block_entries": total_block_entries} + return {} diff --git a/scram/route_manager/tests/test_performance.py b/scram/route_manager/tests/test_performance.py index e12eceed..a1e83a0a 100644 --- a/scram/route_manager/tests/test_performance.py +++ b/scram/route_manager/tests/test_performance.py @@ -26,30 +26,31 @@ def setUp(self): self.atype, _ = ActionType.objects.get_or_create(name="block") routes = [Route(route=self.fake.unique.ipv4_public()) for x in range(self.NUM_ENTRIES)] created_routes = Route.objects.bulk_create(routes) - entries = [Entry(route=route, actiontype=self.atype) for route in created_routes] + entries = [Entry(route=route, actiontype=self.atype, is_active=True) for route in created_routes] Entry.objects.bulk_create(entries) def test_home_page(self): - """Home page requires 8 queries. + """Home page requires 11 queries. 1. create transaction 2. lookup session 3. lookup user 4. filter available actiontypes - 5. count entries with actiontype=1 + 5. count entries with actiontype=1 and is_active 6. count by user - 7. first page for actiontype=1 - 8. close transaction - + 7. context processor active_count active blocks + 8. context processor active_count all blocks + 9. first page for actiontype=1 + 10. close transaction """ - with self.assertNumQueries(8): + with self.assertNumQueries(10): start = time.time() self.client.get(reverse("route_manager:home")) time_taken = time.time() - start self.assertLess(time_taken, 1, "Home page took longer than 1 second") def test_entry_view(self): - """Viewing an entry requires 6 queries. + """Viewing an entry requires 8 queries. 1. create transaction savepoint 2. lookup session @@ -57,9 +58,10 @@ def test_entry_view(self): 4. get entry 5. rollback to savepoint 6. release transaction savepoint - + 7. context processor active_count active blocks + 8. context processor active_count all blocks """ - with self.assertNumQueries(6): + with self.assertNumQueries(8): start = time.time() self.client.get(reverse("route_manager:detail", kwargs={"pk": 9999})) time_taken = time.time() - start diff --git a/scram/route_manager/views.py b/scram/route_manager/views.py index ff282f7e..33be69ab 100644 --- a/scram/route_manager/views.py +++ b/scram/route_manager/views.py @@ -35,10 +35,10 @@ def home_page(request, prefilter=None): readwrite = False context = {"entries": {}, "readwrite": readwrite} for at in ActionType.objects.all(): - queryset = prefilter.filter(actiontype=at).order_by("-pk") + queryset_active = prefilter.filter(actiontype=at, is_active=True) context["entries"][at] = { - "objs": queryset[:num_entries], - "total": queryset.count(), + "objs": queryset_active[:num_entries], + "active": queryset_active.count(), } if settings.AUTOCREATE_ADMIN: diff --git a/scram/templates/navbar.html b/scram/templates/navbar.html index f47e4730..bee63472 100644 --- a/scram/templates/navbar.html +++ b/scram/templates/navbar.html @@ -27,6 +27,9 @@
  • {{ user.name }}
    • +
  • + Blocks:
    {{ active_block_entries }}     +
  • {% csrf_token %} @@ -34,6 +37,9 @@
    • {% else %} +
  • + Blocks:
    {{ active_block_entries }}     +
  • Login
    • {% endif %} diff --git a/scram/templates/route_manager/home.html b/scram/templates/route_manager/home.html index 0cc04231..fefffcaa 100644 --- a/scram/templates/route_manager/home.html +++ b/scram/templates/route_manager/home.html @@ -35,7 +35,7 @@

    SCRAM

    {% if actiontype.available %}

    {{ actiontype.name }} - {{ entry_list.total }} + {{ entry_list.active }}

    {% for entry in entry_list.objs %} {% if actiontype.name|lower == entry.actiontype|lower %}