From c252cf46787d0076b546d263c8be2b2325b9f395 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Fri, 2 Jan 2026 13:57:33 +0000 Subject: [PATCH 01/27] Add ooniapi citizenlab service to load balancer --- tf/modules/ooniapi_frontend/main.tf | 21 +++++++++++++++++++++ tf/modules/ooniapi_frontend/variables.tf | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 93de70fc..8266c465 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -457,3 +457,24 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } } } + +resource "aws_lb_listener_rule" "ooniapi_citizenlab_rule" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 143 + + action { + type = "forward" + target_group_arn = var.ooniapi_citizenlab_target_group_arn + } + condition { + path_pattern { + values = [ + "/api/_/url-submission/test-list/*", + "/api/_/url-priorities/list", + "/api/_/url-priorities/update", + "/api/v1/url-submission/submit", + "/api/v1/url-submission/update-url", + ] + } + } +} diff --git a/tf/modules/ooniapi_frontend/variables.tf b/tf/modules/ooniapi_frontend/variables.tf index d4ec3dd0..2c676694 100644 --- a/tf/modules/ooniapi_frontend/variables.tf +++ b/tf/modules/ooniapi_frontend/variables.tf @@ -37,6 +37,10 @@ variable "ooniapi_oonimeasurements_target_group_arn" { default = null } +variable "ooniapi_citizenlab_target_group_arn" { + description = "arn for the target group of the citizenlab service" +} + variable "dns_zone_ooni_io" { description = "id of the DNS zone for ooni_io" } From a453fc1de8e71f2129bafb7a8a40f73c7f3ee39a Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 10:09:00 +0100 Subject: [PATCH 02/27] Add ec2 t3a.nano for citizenlab test-lists api --- tf/environments/dev/main.tf | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 1aca9365..50a58900 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1087,6 +1087,66 @@ module "ooniapi_oonimeasurements" { ) } +### Tier2 Citizenlab service +module "ooni_citizenlab" { + source = "../../modules/ec2" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3a.nano" + + name = "oonictzlab" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + // API endpoint + from_port = 443, + to_port = 443, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + // For the prometheus proxy: + from_port = 9200, + to_port = 9200, + protocol = "tcp" + cidr_blocks = [for ip in flatten(data.dns_a_record_set.monitoring_host.*.addrs) : "${tostring(ip)}/32"] + }, { + from_port = 9100, + to_port = 9100, + protocol = "tcp" + cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"] + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"] + }] + + sg_prefix = "ooniciti" + tg_prefix = "citi" + + tags = merge( + local.tags, + { Name = "ooni-tier2-citizenlab" } + ) +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { From a513892755e0545b5f808e27ea69f8b5ee633235 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 10:51:23 +0100 Subject: [PATCH 03/27] Add DNS alias for citizenlab.{env}.ooni.io --- tf/environments/dev/main.tf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 50a58900..b958e520 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1147,6 +1147,17 @@ module "ooni_citizenlab" { ) } +resource "aws_route53_record" "citizenlab_alias" { + zone_id = local.dns_zone_ooni_io + name = "citizenlab.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + + records = [ + module.ooni_citizenlab.aws_instance_public_dns + ] +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { From 8b7d0a580b591335353cff9e584594015c242282 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 13:19:46 +0100 Subject: [PATCH 04/27] Add disk_size to citizenlab instance --- tf/environments/dev/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index b958e520..5a591a09 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1141,6 +1141,8 @@ module "ooni_citizenlab" { sg_prefix = "ooniciti" tg_prefix = "citi" + disk_size = 20 + tags = merge( local.tags, { Name = "ooni-tier2-citizenlab" } From 08f7df2ca826c30832bacce21d3b8f4f12038a47 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 13:34:40 +0100 Subject: [PATCH 05/27] Add citizenlab ansible role --- ansible/roles/citizenlab/defaults/main.yml | 8 + ansible/roles/citizenlab/handlers/main.yml | 27 +++ ansible/roles/citizenlab/tasks/main.yml | 168 ++++++++++++++++++ .../citizenlab/templates/citizenlab.conf | 19 ++ 4 files changed, 222 insertions(+) create mode 100644 ansible/roles/citizenlab/defaults/main.yml create mode 100644 ansible/roles/citizenlab/handlers/main.yml create mode 100644 ansible/roles/citizenlab/tasks/main.yml create mode 100644 ansible/roles/citizenlab/templates/citizenlab.conf diff --git a/ansible/roles/citizenlab/defaults/main.yml b/ansible/roles/citizenlab/defaults/main.yml new file mode 100644 index 00000000..5c8c6209 --- /dev/null +++ b/ansible/roles/citizenlab/defaults/main.yml @@ -0,0 +1,8 @@ +tls_cert_dir: /var/lib/dehydrated/certs + +# citizenlab user +citizenlab_user: citizenlab +citizenlab_home: "/opt/{{ citizenlab_user }}" + +# citizenlab settings +clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" diff --git a/ansible/roles/citizenlab/handlers/main.yml b/ansible/roles/citizenlab/handlers/main.yml new file mode 100644 index 00000000..594ebfe9 --- /dev/null +++ b/ansible/roles/citizenlab/handlers/main.yml @@ -0,0 +1,27 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart docker + tags: docker + ansible.builtin.systemd_service: + name: docker + state: restarted \ No newline at end of file diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml new file mode 100644 index 00000000..98266485 --- /dev/null +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -0,0 +1,168 @@ +--- +# For prometheus scrape requests +- name: Flush all handlers + meta: flush_handlers + +- name: Allow traffic on port 9100 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9100.nft + create: yes + block: | + add rule inet filter input tcp dport 9100 counter accept comment "node exporter" + notify: + - reload nftables + +# For statsd importer metrics +- name: Flush all handlers + meta: flush_handlers + +- name: Allow traffic on port 9102 + become: true + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9102.nft + create: yes + block: | + add rule inet filter input tcp dport 9102 counter accept comment "node exporter" + notify: + - reload nftables + +# For incoming citizenlab traffic +- name: Allow traffic on port 443 + become: true + tags: citizenlab + blockinfile: + path: /etc/ooni/nftables/tcp/443.nft + create: yes + block: | + add rule inet filter input tcp dport 443 counter accept comment "citizenlab" + notify: + - reload nftables + +# Docker seems to have problems with nftables, so this command will translate all iptables +# commands to nftables commands +# - name: Update alternatives for iptables +# tags: docker +# become: yes +# ansible.builtin.command: "update-alternatives --set iptables /usr/sbin/iptables-nft" +# notify: +# - restart docker + +# - name: Update alternatives for iptables +# tags: docker +# become: yes +# ansible.builtin.command: "update-alternatives --set ip6tables /usr/sbin/ip6tables-nft" +# notify: +# - restart docker + +# - name: Flush all handlers # Required to apply iptables settings before docker runs +# meta: flush_handlers + +### Create citizenlab user +- name: Ensure the citizenlab group exists + ansible.builtin.group: + name: "{{ citizenlab_user }}" + state: present + gid: "1420" + become: yes + +- name: Create the citizenlab user + ansible.builtin.user: + name: "{{ citizenlab_user }}" + home: "{{ citizenlab_home }}" + shell: "/bin/bash" + group: "{{ citizenlab_user }}" + create_home: yes + system: yes + uid: "1420" + become: yes + +- name: Set ownership of the citizenlab directory + ansible.builtin.file: + path: "{{ citizenlab_home }}" + owner: "{{ citizenlab_user }}" + group: "{{ citizenlab_user }}" + state: directory + mode: "0700" + become: yes + +### Run citizenlab +- name: Make sure that the citizenlab configuration directory exists + ansible.builtin.file: + path: /opt/{{citizenlab_user}}/backend/citizenlab/ + state: directory + mode: "0700" + owner: "{{citizenlab_user}}" + group: "{{citizenlab_user}}" + +- name: Create configuration file + tags: citizenlab + template: + src: templates/citizenlab.conf + dest: "/opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf" + mode: "0400" + owner: "{{citizenlab_user}}" + become: yes + +- name: Ensure ooniapi directory existence + ansible.builtin.file: + path: /var/lib/ooniapi + state: directory + mode: "0711" + owner: "{{citizenlab_user}}" + group: "{{citizenlab_user}}" + +- name: Ensure citizenlab var dir exists + ansible.builtin.file: + path: /var/lib/citizenlab + state: directory + mode: "0700" + owner: "{{citizenlab_user}}" + group: "{{citizenlab_user}}" + +- name: Allow nginx access to the spool dir + become: true + ansible.builtin.user: + name: nginx + groups: "{{citizenlab_user}}" + append: yes + +- name: Get UID of a specific user + command: id -u {{citizenlab_user}} + register: user_uid + changed_when: false + +- name: Get GID of a specific user + command: id -g {{citizenlab_user}} + register: user_gid + changed_when: false + +- name: Ensure citizenlab is running + community.docker.docker_container: + name: citizenlab + image: ooni/citizenlab:v0.2 + state: started + user: "{{user_uid.stdout}}:{{user_gid.stdout}}" + # use network mode = host to allow traffic from citizenlab to the statsd exporter without + # creating a network with redirection rules to match the ports + network_mode: host + # published_ports: # unused on network_mode: host + # - "8472:8472" + volumes: + - /opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf:/etc/ooni/citizenlab.conf + - /var/lib/ooniapi:/var/lib/ooniapi + - /var/lib/citizenlab:/var/lib/citizenlab + tags: + - citizenlab + +- name: Ensure the statsd to prometheus exporter is running + community.docker.docker_container: + name: statsd-exporter + image: prom/statsd-exporter:v0.28.0 + state: started + published_ports: + - "9102:9102" # for /metrics + - "8125:9125" # for reporting metrics + - "8125:9125/udp" diff --git a/ansible/roles/citizenlab/templates/citizenlab.conf b/ansible/roles/citizenlab/templates/citizenlab.conf new file mode 100644 index 00000000..54ec9d7e --- /dev/null +++ b/ansible/roles/citizenlab/templates/citizenlab.conf @@ -0,0 +1,19 @@ +[DEFAULT] +# Collector hostnames, comma separated +collectors = localhost + + +{% if psql_uri is defined %} +# The password is already made public +db_uri = {{ psql_uri }} +{% else %} +db_uri = +{% endif %} + +# S3 access credentials +# Currently unused +s3_access_key = +s3_secret_key = + + +clickhouse_url = {{clickhouse_url}} From ec601edcc7b94842d84867c4d8801e3921d615e4 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 13:37:18 +0100 Subject: [PATCH 06/27] Add port 80 rule for dehydrated --- tf/environments/dev/main.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5a591a09..d998ec6a 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1108,6 +1108,11 @@ module "ooni_citizenlab" { protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { + from_port = 80, # for dehydrated challenge + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { // API endpoint from_port = 443, to_port = 443, From 8c2b839335d34df455be2903055b99f9ade4f7a9 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 13:43:05 +0100 Subject: [PATCH 07/27] add citizenlab hosts to inventory --- ansible/deploy-citizenlab.yml | 23 +++++++++++++++++++++++ ansible/inventory | 2 ++ 2 files changed, 25 insertions(+) create mode 100644 ansible/deploy-citizenlab.yml diff --git a/ansible/deploy-citizenlab.yml b/ansible/deploy-citizenlab.yml new file mode 100644 index 00000000..57ea9acb --- /dev/null +++ b/ansible/deploy-citizenlab.yml @@ -0,0 +1,23 @@ +--- +- name: Deploy citizenlab + hosts: + - citizenlab.dev.ooni.io + - citizenlab.prod.ooni.io + become: true + roles: + - role: bootstrap + - role: nginx + - role: prometheus_node_exporter + vars: + use_https: false + node_exporter_port: 9100 + node_exporter_host: "0.0.0.0" + prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:9100/metrics + - role: geerlingguy.docker + docker_users: + - citizenlab + - ubuntu + docker_package_state: latest + - role: citizenlab diff --git a/ansible/inventory b/ansible/inventory index 7437f1af..93eebb31 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -49,3 +49,5 @@ fastpath.prod.ooni.io anonc.dev.ooni.io jumphost.dev.ooni.io jumphost.prod.ooni.io +citizenlab.dev.ooni.io +citizenlab.prod.ooni.io From de5e35e865817ecaa9441e43502e12f245f405ff Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 13:43:19 +0100 Subject: [PATCH 08/27] add citizenlab hosts to prometheus role --- .../roles/prometheus/templates/prometheus.yml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index 56f53a72..dc97adbb 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -370,4 +370,50 @@ scrape_configs: replacement: "/$2" target_label: "__metrics_path__" action: "replace" + + - job_name: "citizenlab" + static_configs: + - targets: + - citizenlab.dev.ooni.io:9102 + - citizenlab.prod.ooni.io:9102 + scrape_interval: 5s + scheme: https + relabel_configs: # Change the host to the proxy host with relabeling + # Store ip in ecs_host + - source_labels: [__address__] + regex: "([a-z\\.]+):([0-9]+)" # :" + replacement: "$1" + target_label: "ec2_host" + action: "replace" + # Extract environment from address + - source_labels: [__address__] + regex: ".*(dev|prod).*" + replacement: "$1" + target_label: "env" + action: "replace" + # Store the full adress with path in proxy_host + - source_labels: [__address__] + regex: "([a-z\\.]+):([0-9]+)" # : + replacement: "{{monitoring_proxy_host}}:9200/${1}/${2}/metrics" # proxy.org:9200///metrics + target_label: "__proxy_host" + action: "replace" + # Change the environment part in proxy host + - source_labels: [__proxy_host, env] + separator: ";" + regex: "([^;]*)ENV([^;]*);(.*)" # __proxy_host;env + replacement: "$1$3$2" + target_label: "__proxy_host" + action: "replace" + # Change the address where to send the scrape request to + - source_labels: [__proxy_host] + regex: "([^/]*)/(.*)" + replacement: "$1" + target_label: "__address__" + action: "replace" + # Change the metrics path to include ip address and /metrics path + - source_labels: [__proxy_host] + regex: "([^/]*)/(.*)" + replacement: "/$2" + target_label: "__metrics_path__" + action: "replace" ... From c84435cee5e16e3a525cb0ca9fc1ac88b673b54f Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 14:13:23 +0100 Subject: [PATCH 09/27] Add citizenlab playbook to deploy-tier2 --- ansible/deploy-tier2.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/deploy-tier2.yml b/ansible/deploy-tier2.yml index cff8d695..79df612e 100644 --- a/ansible/deploy-tier2.yml +++ b/ansible/deploy-tier2.yml @@ -9,6 +9,9 @@ - name: Include notebook playbook ansible.builtin.import_playbook: deploy-notebook.yml +- name: Include citizenlab playbook + ansible.builtin.import_playbook: deploy-citizenlab.yml + # commented out due to the fact it requires manual config of ~/.ssh/config #- name: Setup codesign box # hosts: codesign-box From b4e77ebf15f704546bfa711117c9f3ea4558e858 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 14:23:46 +0100 Subject: [PATCH 10/27] Add citizenlab_builder to dev --- tf/environments/dev/main.tf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index d998ec6a..360e12a2 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1165,6 +1165,22 @@ resource "aws_route53_record" "citizenlab_alias" { ] } +module "citizenlab_builder" { + source = "../../modules/ooni_docker_build" + trigger_tag = "" + + service_name = "citizenlab" + repo = "ooni/backend" + branch_name = "add_citizenlab_url_management_with_porcelain" + buildspec_path = "ooniapi/services/citizenlab/buildspec.yml" + trigger_path = "ooniapi/services/citizenlab/**" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + #### OONI Tier0 API Frontend module "ooniapi_frontend" { From 1216c3fe8f84a4fec9009cef71ed3bc2ff19b174 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 14:30:32 +0100 Subject: [PATCH 11/27] use citizenlab version from hatch version --- ansible/roles/citizenlab/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index 98266485..c7384bcb 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -142,7 +142,7 @@ - name: Ensure citizenlab is running community.docker.docker_container: name: citizenlab - image: ooni/citizenlab:v0.2 + image: ooni/citizenlab:v0.1.0rc0 state: started user: "{{user_uid.stdout}}:{{user_gid.stdout}}" # use network mode = host to allow traffic from citizenlab to the statsd exporter without From 90f3ee3809c56f9f1534c867161fed723fef0e70 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 20 Jan 2026 14:59:28 +0100 Subject: [PATCH 12/27] Add citizenlab service to mermaid diagram --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index fa9b8c7a..45bb0076 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ flowchart TB apiorg([api.ooni.org])-->alb apiio([api.ooni.io])-->backend ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)] + cz[Citizenlab API EC2]<-->ch + subgraph Hetzner backend[OONI Backend Monolith]<-->ch monitoring[Monitoring host] @@ -15,6 +17,7 @@ flowchart TB subgraph AWS alb[API Load Balancer]<-->ecs alb-->backend + alb-->cz ecs<-->s3[(OONI S3 Buckets)] s3<-->backend end From ba441f0ce347b3af5ae82be83d1e46042ae67397 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 27 Jan 2026 11:25:05 +0100 Subject: [PATCH 13/27] fix whitespace --- ansible/deploy-citizenlab.yml | 2 +- ansible/roles/citizenlab/handlers/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/deploy-citizenlab.yml b/ansible/deploy-citizenlab.yml index 57ea9acb..99dc234a 100644 --- a/ansible/deploy-citizenlab.yml +++ b/ansible/deploy-citizenlab.yml @@ -12,7 +12,7 @@ use_https: false node_exporter_port: 9100 node_exporter_host: "0.0.0.0" - prometheus_nginx_proxy_config: + prometheus_nginx_proxy_config: - location: /metrics/node_exporter proxy_pass: http://127.0.0.1:9100/metrics - role: geerlingguy.docker diff --git a/ansible/roles/citizenlab/handlers/main.yml b/ansible/roles/citizenlab/handlers/main.yml index 594ebfe9..5c8f17dd 100644 --- a/ansible/roles/citizenlab/handlers/main.yml +++ b/ansible/roles/citizenlab/handlers/main.yml @@ -19,9 +19,9 @@ ansible.builtin.systemd_service: name: nftables state: reloaded - + - name: restart docker tags: docker ansible.builtin.systemd_service: name: docker - state: restarted \ No newline at end of file + state: restarted From 746b086661ed7dd688a0686f21a099f950d0efac Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 27 Jan 2026 11:25:24 +0100 Subject: [PATCH 14/27] fetch clickhouse_url from aws secrets --- ansible/roles/citizenlab/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/citizenlab/defaults/main.yml b/ansible/roles/citizenlab/defaults/main.yml index 5c8c6209..6842b38c 100644 --- a/ansible/roles/citizenlab/defaults/main.yml +++ b/ansible/roles/citizenlab/defaults/main.yml @@ -5,4 +5,4 @@ citizenlab_user: citizenlab citizenlab_home: "/opt/{{ citizenlab_user }}" # citizenlab settings -clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') }}@clickhouseproxy.dev.ooni.io/oonitest" +clickhouse_url: "" # fetch from aws secrets From b01f4f934ceb70ff52dfc3b980cbfeaee61c0658 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 27 Jan 2026 11:29:28 +0100 Subject: [PATCH 15/27] dont set specific uid, gid for user citizenlab --- ansible/roles/citizenlab/tasks/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index c7384bcb..3069be11 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -65,7 +65,6 @@ ansible.builtin.group: name: "{{ citizenlab_user }}" state: present - gid: "1420" become: yes - name: Create the citizenlab user @@ -76,7 +75,6 @@ group: "{{ citizenlab_user }}" create_home: yes system: yes - uid: "1420" become: yes - name: Set ownership of the citizenlab directory From 3068b2da1c6faf0253f813a1116bb91fabb88dc5 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 27 Jan 2026 13:25:08 +0100 Subject: [PATCH 16/27] rename ooni_citizenlab to ooniapi_citizenlab, add target_group_arn from aws_instance_id --- tf/environments/dev/main.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 360e12a2..1f381fbf 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1088,7 +1088,7 @@ module "ooniapi_oonimeasurements" { } ### Tier2 Citizenlab service -module "ooni_citizenlab" { +module "ooniapi_citizenlab" { source = "../../modules/ec2" stage = local.environment @@ -1161,7 +1161,7 @@ resource "aws_route53_record" "citizenlab_alias" { ttl = 300 records = [ - module.ooni_citizenlab.aws_instance_public_dns + module.ooniapi_citizenlab.aws_instance_public_dns ] } @@ -1195,6 +1195,7 @@ module "ooniapi_frontend" { ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id + ooniapi_citizenlab_target_group_arn = module.ooniapi_citizenlab.aws_instance_id ooniapi_service_security_groups = [ module.ooniapi_cluster.web_security_group_id, From c7b1fac1e7116a405726c415e6bf7393ec370f29 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 27 Jan 2026 13:54:25 +0100 Subject: [PATCH 17/27] Disable load balancer rule for citizenlab in ooniapi_frontend --- tf/modules/ooniapi_frontend/main.tf | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 8266c465..70635854 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -458,23 +458,23 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } } -resource "aws_lb_listener_rule" "ooniapi_citizenlab_rule" { - listener_arn = aws_alb_listener.ooniapi_listener_https.arn - priority = 143 - - action { - type = "forward" - target_group_arn = var.ooniapi_citizenlab_target_group_arn - } - condition { - path_pattern { - values = [ - "/api/_/url-submission/test-list/*", - "/api/_/url-priorities/list", - "/api/_/url-priorities/update", - "/api/v1/url-submission/submit", - "/api/v1/url-submission/update-url", - ] - } - } -} +#resource "aws_lb_listener_rule" "ooniapi_citizenlab_rule" { +# listener_arn = aws_alb_listener.ooniapi_listener_https.arn +# priority = 143 +# +# action { +# type = "forward" +# target_group_arn = var.ooniapi_citizenlab_target_group_arn +# } +# condition { +# path_pattern { +# values = [ +# "/api/_/url-submission/test-list/*", +# "/api/_/url-priorities/list", +# "/api/_/url-priorities/update", +# "/api/v1/url-submission/submit", +# "/api/v1/url-submission/update-url", +# ] +# } +# } +#} From cfe92c254230384f2f20d88ed48afd288bd19c51 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 29 Jan 2026 16:08:54 +0100 Subject: [PATCH 18/27] use correct image name ooni/api-citizenlab --- ansible/roles/citizenlab/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index 3069be11..4dcc9f2f 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -140,7 +140,7 @@ - name: Ensure citizenlab is running community.docker.docker_container: name: citizenlab - image: ooni/citizenlab:v0.1.0rc0 + image: ooni/api-citizenlab:v0.1.0rc0 state: started user: "{{user_uid.stdout}}:{{user_gid.stdout}}" # use network mode = host to allow traffic from citizenlab to the statsd exporter without From 71b7cea8553d15b1bb9ca64247a2671978293a83 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Thu, 29 Jan 2026 16:09:11 +0100 Subject: [PATCH 19/27] remove unused statsd from citizenlab --- ansible/roles/citizenlab/tasks/main.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index 4dcc9f2f..ebec8e2a 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -154,13 +154,3 @@ - /var/lib/citizenlab:/var/lib/citizenlab tags: - citizenlab - -- name: Ensure the statsd to prometheus exporter is running - community.docker.docker_container: - name: statsd-exporter - image: prom/statsd-exporter:v0.28.0 - state: started - published_ports: - - "9102:9102" # for /metrics - - "8125:9125" # for reporting metrics - - "8125:9125/udp" From 0f65c8f828742854a5f432ad5858851948ac3fbb Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 11:59:16 +0100 Subject: [PATCH 20/27] citizenlab: use latest docker image at deployment --- ansible/roles/citizenlab/tasks/main.yml | 2 +- tf/environments/dev/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index ebec8e2a..aef0424f 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -140,7 +140,7 @@ - name: Ensure citizenlab is running community.docker.docker_container: name: citizenlab - image: ooni/api-citizenlab:v0.1.0rc0 + image: ooni/api-citizenlab:latest state: started user: "{{user_uid.stdout}}:{{user_gid.stdout}}" # use network mode = host to allow traffic from citizenlab to the statsd exporter without diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 1f381fbf..2605b700 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1171,7 +1171,7 @@ module "citizenlab_builder" { service_name = "citizenlab" repo = "ooni/backend" - branch_name = "add_citizenlab_url_management_with_porcelain" + branch_name = "add_citizenlab_url_management_with_porcelain.3" buildspec_path = "ooniapi/services/citizenlab/buildspec.yml" trigger_path = "ooniapi/services/citizenlab/**" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn From 5cf22e02784f1087933cec3d0e2f61b61a38d52e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 11:59:52 +0100 Subject: [PATCH 21/27] citizenlab: use larger instance --- tf/environments/dev/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 2605b700..0b983a11 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1099,7 +1099,7 @@ module "ooniapi_citizenlab" { dns_zone_ooni_io = local.dns_zone_ooni_io key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t3a.nano" + instance_type = "t3a.micro" name = "oonictzlab" ingress_rules = [{ From ec4da4b7fa967d00130cb17d9b933b298dae749e Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 12:20:09 +0100 Subject: [PATCH 22/27] listen on port 80, not 443, remove dehydrated --- ansible/roles/citizenlab/defaults/main.yml | 2 -- ansible/roles/citizenlab/tasks/main.yml | 25 +++------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/ansible/roles/citizenlab/defaults/main.yml b/ansible/roles/citizenlab/defaults/main.yml index 6842b38c..e049a189 100644 --- a/ansible/roles/citizenlab/defaults/main.yml +++ b/ansible/roles/citizenlab/defaults/main.yml @@ -1,5 +1,3 @@ -tls_cert_dir: /var/lib/dehydrated/certs - # citizenlab user citizenlab_user: citizenlab citizenlab_home: "/opt/{{ citizenlab_user }}" diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index aef0424f..e4a953d9 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -30,36 +30,17 @@ - reload nftables # For incoming citizenlab traffic -- name: Allow traffic on port 443 +- name: Allow traffic on port 80 become: true tags: citizenlab blockinfile: - path: /etc/ooni/nftables/tcp/443.nft + path: /etc/ooni/nftables/tcp/80.nft create: yes block: | - add rule inet filter input tcp dport 443 counter accept comment "citizenlab" + add rule inet filter input tcp dport 80 counter accept comment "citizenlab" notify: - reload nftables -# Docker seems to have problems with nftables, so this command will translate all iptables -# commands to nftables commands -# - name: Update alternatives for iptables -# tags: docker -# become: yes -# ansible.builtin.command: "update-alternatives --set iptables /usr/sbin/iptables-nft" -# notify: -# - restart docker - -# - name: Update alternatives for iptables -# tags: docker -# become: yes -# ansible.builtin.command: "update-alternatives --set ip6tables /usr/sbin/ip6tables-nft" -# notify: -# - restart docker - -# - name: Flush all handlers # Required to apply iptables settings before docker runs -# meta: flush_handlers - ### Create citizenlab user - name: Ensure the citizenlab group exists ansible.builtin.group: From 89c34eae849cd2a2cedfdd75173683eff66d9e70 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 12:20:35 +0100 Subject: [PATCH 23/27] remove nginx related task --- ansible/roles/citizenlab/tasks/main.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ansible/roles/citizenlab/tasks/main.yml b/ansible/roles/citizenlab/tasks/main.yml index e4a953d9..e589f48f 100644 --- a/ansible/roles/citizenlab/tasks/main.yml +++ b/ansible/roles/citizenlab/tasks/main.yml @@ -101,13 +101,6 @@ owner: "{{citizenlab_user}}" group: "{{citizenlab_user}}" -- name: Allow nginx access to the spool dir - become: true - ansible.builtin.user: - name: nginx - groups: "{{citizenlab_user}}" - append: yes - - name: Get UID of a specific user command: id -u {{citizenlab_user}} register: user_uid From b8163a91c7bc4308c767185d4487a56c8b3c4fb0 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 12:27:55 +0100 Subject: [PATCH 24/27] remove nginx role from deploy-citizenlab.yml --- ansible/deploy-citizenlab.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ansible/deploy-citizenlab.yml b/ansible/deploy-citizenlab.yml index 99dc234a..fcf7a00e 100644 --- a/ansible/deploy-citizenlab.yml +++ b/ansible/deploy-citizenlab.yml @@ -6,15 +6,11 @@ become: true roles: - role: bootstrap - - role: nginx - role: prometheus_node_exporter vars: use_https: false node_exporter_port: 9100 node_exporter_host: "0.0.0.0" - prometheus_nginx_proxy_config: - - location: /metrics/node_exporter - proxy_pass: http://127.0.0.1:9100/metrics - role: geerlingguy.docker docker_users: - citizenlab From b4263d4ff212fba3f26fb433b92fd2de533669a7 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 12:32:37 +0100 Subject: [PATCH 25/27] remove nginx handlers from citizenlab --- ansible/roles/citizenlab/handlers/main.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ansible/roles/citizenlab/handlers/main.yml b/ansible/roles/citizenlab/handlers/main.yml index 5c8f17dd..d44c00f4 100644 --- a/ansible/roles/citizenlab/handlers/main.yml +++ b/ansible/roles/citizenlab/handlers/main.yml @@ -1,19 +1,3 @@ -- name: test nginx config - command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf - listen: - - restart nginx - - reload nginx - -- name: restart nginx - service: - name: nginx - state: restarted - -- name: reload nginx - service: - name: nginx - state: reloaded - - name: reload nftables tags: nftables ansible.builtin.systemd_service: From 6db75a80415b1b7ef88b046152ee21d2119d35cd Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 12:40:59 +0100 Subject: [PATCH 26/27] add citizenlab.{env}.ooni.io to ooniapi_frontend_alternative_domains --- tf/environments/dev/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 0b983a11..6cef7efe 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -1222,6 +1222,7 @@ locals { "oonirun.${local.environment}.ooni.io" : local.dns_zone_ooni_io, "oonimeasurements.${local.environment}.ooni.io" : local.dns_zone_ooni_io, "8.th.dev.ooni.io" : local.dns_zone_ooni_io, + "citizenlab.${local.environment}.ooni.io" : local.dns_zone_ooni_io, } ooniapi_frontend_main_domain_name = "api.${local.environment}.ooni.io" ooniapi_frontend_main_domain_name_zone_id = local.dns_zone_ooni_io From cd9d2ee93c2b91291a9236289f8ffd3045910564 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Wed, 4 Feb 2026 12:41:30 +0100 Subject: [PATCH 27/27] uncomment citizenlab load balancer rules --- tf/modules/ooniapi_frontend/main.tf | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index 70635854..8266c465 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -458,23 +458,23 @@ resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" { } } -#resource "aws_lb_listener_rule" "ooniapi_citizenlab_rule" { -# listener_arn = aws_alb_listener.ooniapi_listener_https.arn -# priority = 143 -# -# action { -# type = "forward" -# target_group_arn = var.ooniapi_citizenlab_target_group_arn -# } -# condition { -# path_pattern { -# values = [ -# "/api/_/url-submission/test-list/*", -# "/api/_/url-priorities/list", -# "/api/_/url-priorities/update", -# "/api/v1/url-submission/submit", -# "/api/v1/url-submission/update-url", -# ] -# } -# } -#} +resource "aws_lb_listener_rule" "ooniapi_citizenlab_rule" { + listener_arn = aws_alb_listener.ooniapi_listener_https.arn + priority = 143 + + action { + type = "forward" + target_group_arn = var.ooniapi_citizenlab_target_group_arn + } + condition { + path_pattern { + values = [ + "/api/_/url-submission/test-list/*", + "/api/_/url-priorities/list", + "/api/_/url-priorities/update", + "/api/v1/url-submission/submit", + "/api/v1/url-submission/update-url", + ] + } + } +}