Add citizenlab ec2 service to dev

README.md

@@ -7,6 +7,8 @@ flowchart TB
     apiorg([api.ooni.org])-->alb
     apiio([api.ooni.io])-->backend
     ecs[Backend API ECS]<-->ch[(Clickhouse Cluster)]
+    cz[Citizenlab API EC2]<-->ch
+
     subgraph Hetzner
         backend[OONI Backend Monolith]<-->ch
         monitoring[Monitoring host]
@@ -15,6 +17,7 @@ flowchart TB
     subgraph AWS
     alb[API Load Balancer]<-->ecs
     alb-->backend
+    alb-->cz
     ecs<-->s3[(OONI S3 Buckets)]
     s3<-->backend
     end

ansible/deploy-citizenlab.yml

@@ -0,0 +1,19 @@
+---
+- name: Deploy citizenlab
+  hosts:
+    - citizenlab.dev.ooni.io
+    - citizenlab.prod.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: prometheus_node_exporter
+      vars:
+        use_https: false
+        node_exporter_port: 9100
+        node_exporter_host: "0.0.0.0"
+    - role: geerlingguy.docker
+      docker_users:
+        - citizenlab
+        - ubuntu
+      docker_package_state: latest
+    - role: citizenlab

ansible/deploy-tier2.yml

@@ -9,6 +9,9 @@
 - name: Include notebook playbook
   ansible.builtin.import_playbook: deploy-notebook.yml
 
+- name: Include citizenlab playbook
+  ansible.builtin.import_playbook: deploy-citizenlab.yml
+
 # commented out due to the fact it requires manual config of ~/.ssh/config
 #- name: Setup codesign box
 #  hosts: codesign-box

ansible/inventory

@@ -49,3 +49,5 @@ fastpath.prod.ooni.io
 anonc.dev.ooni.io
 jumphost.dev.ooni.io
 jumphost.prod.ooni.io
+citizenlab.dev.ooni.io
+citizenlab.prod.ooni.io

ansible/roles/citizenlab/defaults/main.yml

@@ -0,0 +1,6 @@
+# citizenlab user
+citizenlab_user: citizenlab
+citizenlab_home: "/opt/{{ citizenlab_user }}"
+
+# citizenlab settings
+clickhouse_url: "" # fetch from aws secrets

ansible/roles/citizenlab/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded
+
+- name: restart docker
+  tags: docker
+  ansible.builtin.systemd_service:
+    name: docker
+    state: restarted

ansible/roles/citizenlab/tasks/main.yml

@@ -0,0 +1,130 @@
+---
+# For prometheus scrape requests
+- name: Flush all handlers
+  meta: flush_handlers
+
+- name: Allow traffic on port 9100
+  become: true
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9100.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9100 counter accept comment "node exporter"
+  notify:
+   - reload nftables
+
+# For statsd importer metrics
+- name: Flush all handlers
+  meta: flush_handlers
+
+- name: Allow traffic on port 9102
+  become: true
+  tags: prometheus-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9102.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9102 counter accept comment "node exporter"
+  notify:
+   - reload nftables
+
+# For incoming citizenlab traffic
+- name: Allow traffic on port 80
+  become: true
+  tags: citizenlab
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/80.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 80 counter accept comment "citizenlab"
+  notify:
+   - reload nftables
+
+### Create citizenlab user
+- name: Ensure the citizenlab group exists
+  ansible.builtin.group:
+    name: "{{ citizenlab_user }}"
+    state: present
+  become: yes
+
+- name: Create the citizenlab user
+  ansible.builtin.user:
+    name: "{{ citizenlab_user }}"
+    home: "{{ citizenlab_home }}"
+    shell: "/bin/bash"
+    group: "{{ citizenlab_user }}"
+    create_home: yes
+    system: yes
+  become: yes
+
+- name: Set ownership of the citizenlab directory
+  ansible.builtin.file:
+    path: "{{ citizenlab_home }}"
+    owner: "{{ citizenlab_user }}"
+    group: "{{ citizenlab_user }}"
+    state: directory
+    mode: "0700"
+  become: yes
+
+### Run citizenlab
+- name: Make sure that the citizenlab configuration directory exists
+  ansible.builtin.file:
+    path: /opt/{{citizenlab_user}}/backend/citizenlab/
+    state: directory
+    mode: "0700"
+    owner: "{{citizenlab_user}}"
+    group: "{{citizenlab_user}}"
+
+- name: Create configuration file
+  tags: citizenlab
+  template:
+    src: templates/citizenlab.conf
+    dest: "/opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf"
+    mode: "0400"
+    owner: "{{citizenlab_user}}"
+  become: yes
+
+- name: Ensure ooniapi directory existence
+  ansible.builtin.file:
+    path: /var/lib/ooniapi
+    state: directory
+    mode: "0711"
+    owner: "{{citizenlab_user}}"
+    group: "{{citizenlab_user}}"
+
+- name: Ensure citizenlab var dir exists
+  ansible.builtin.file:
+    path: /var/lib/citizenlab
+    state: directory
+    mode: "0700"
+    owner: "{{citizenlab_user}}"
+    group: "{{citizenlab_user}}"
+
+- name: Get UID of a specific user
+  command: id -u {{citizenlab_user}}
+  register: user_uid
+  changed_when: false
+
+- name: Get GID of a specific user
+  command: id -g {{citizenlab_user}}
+  register: user_gid
+  changed_when: false
+
+- name: Ensure citizenlab is running
+  community.docker.docker_container:
+    name: citizenlab
+    image: ooni/api-citizenlab:latest
+    state: started
+    user: "{{user_uid.stdout}}:{{user_gid.stdout}}"
+    # use network mode = host to allow traffic from citizenlab to the statsd exporter without
+    # creating a network with redirection rules to match the ports
+    network_mode: host
+    # published_ports: # unused on network_mode: host
+    #   - "8472:8472"
+    volumes:
+      - /opt/{{citizenlab_user}}/backend/citizenlab/citizenlab.conf:/etc/ooni/citizenlab.conf
+      - /var/lib/ooniapi:/var/lib/ooniapi
+      - /var/lib/citizenlab:/var/lib/citizenlab
+  tags:
+    - citizenlab

ansible/roles/citizenlab/templates/citizenlab.conf

@@ -0,0 +1,19 @@
+[DEFAULT]
+# Collector hostnames, comma separated
+collectors = localhost
+
+
+{% if psql_uri is defined %}
+# The password is already made public
+db_uri = {{ psql_uri }}
+{% else %}
+db_uri =
+{% endif %}
+
+# S3 access credentials
+# Currently unused
+s3_access_key =
+s3_secret_key =
+
+
+clickhouse_url = {{clickhouse_url}}

ansible/roles/prometheus/templates/prometheus.yml

@@ -370,4 +370,50 @@ scrape_configs:
         replacement: "/$2"
         target_label: "__metrics_path__"
         action: "replace"
+
+  - job_name: "citizenlab"
+    static_configs:
+    - targets:
+      - citizenlab.dev.ooni.io:9102
+      - citizenlab.prod.ooni.io:9102
+    scrape_interval: 5s
+    scheme: https
+    relabel_configs: # Change the host to the proxy host with relabeling
+      # Store ip in ecs_host
+      - source_labels: [__address__]
+        regex: "([a-z\\.]+):([0-9]+)" # <host>:<port>"
+        replacement: "$1"
+        target_label: "ec2_host"
+        action: "replace"
+      # Extract environment from address
+      - source_labels: [__address__]
+        regex: ".*(dev|prod).*"
+        replacement: "$1"
+        target_label: "env"
+        action: "replace"
+      # Store the full adress with path in proxy_host
+      - source_labels: [__address__]
+        regex: "([a-z\\.]+):([0-9]+)" # <host>:<port>
+        replacement: "{{monitoring_proxy_host}}:9200/${1}/${2}/metrics" # proxy.org:9200/<hostname>/<port>/metrics
+        target_label: "__proxy_host"
+        action: "replace"
+      # Change the environment part in proxy host
+      - source_labels: [__proxy_host, env]
+        separator: ";"
+        regex: "([^;]*)ENV([^;]*);(.*)" # __proxy_host;env
+        replacement: "$1$3$2"
+        target_label: "__proxy_host"
+        action: "replace"
+      # Change the address where to send the scrape request to
+      - source_labels: [__proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "$1"
+        target_label: "__address__"
+        action: "replace"
+      # Change the metrics path to include ip address and /metrics path
+      - source_labels: [__proxy_host]
+        regex: "([^/]*)/(.*)"
+        replacement: "/$2"
+        target_label: "__metrics_path__"
+        action: "replace"
 ...

tf/environments/dev/main.tf

@@ -1087,6 +1087,100 @@ module "ooniapi_oonimeasurements" {
   )
 }
 
+### Tier2 Citizenlab service
+module "ooniapi_citizenlab" {
+  source = "../../modules/ec2"
+
+  stage = local.environment
+
+  vpc_id              = module.network.vpc_id
+  subnet_id           = module.network.vpc_subnet_public[0].id
+  private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block
+  dns_zone_ooni_io    = local.dns_zone_ooni_io
+
+  key_name      = module.adm_iam_roles.oonidevops_key_name
+  instance_type = "t3a.micro"
+
+  name = "oonictzlab"
+  ingress_rules = [{
+    from_port   = 22,
+    to_port     = 22,
+    protocol    = "tcp",
+    cidr_blocks = ["0.0.0.0/0"],
+    }, {
+    from_port   = 80, # for dehydrated challenge
+    to_port     = 80,
+    protocol    = "tcp",
+    cidr_blocks = ["0.0.0.0/0"],
+    }, {
+    // API endpoint
+    from_port   = 443,
+    to_port     = 443,
+    protocol    = "tcp",
+    cidr_blocks = ["0.0.0.0/0"],
+    }, {
+    // For the prometheus proxy:
+    from_port   = 9200,
+    to_port     = 9200,
+    protocol    = "tcp"
+    cidr_blocks = [for ip in flatten(data.dns_a_record_set.monitoring_host.*.addrs) : "${tostring(ip)}/32"]
+    }, {
+    from_port   = 9100,
+    to_port     = 9100,
+    protocol    = "tcp"
+    cidr_blocks = ["${module.ooni_monitoring_proxy.aws_instance_private_ip}/32"]
+  }]
+
+  egress_rules = [{
+    from_port   = 0,
+    to_port     = 0,
+    protocol    = "-1",
+    cidr_blocks = ["0.0.0.0/0"],
+    }, {
+    from_port        = 0,
+    to_port          = 0,
+    protocol         = "-1",
+    ipv6_cidr_blocks = ["::/0"]
+  }]
+
+  sg_prefix = "ooniciti"
+  tg_prefix = "citi"
+
+  disk_size = 20
+
+  tags = merge(
+    local.tags,
+    { Name = "ooni-tier2-citizenlab" }
+  )
+}
+
+resource "aws_route53_record" "citizenlab_alias" {
+  zone_id = local.dns_zone_ooni_io
+  name    = "citizenlab.${local.environment}.ooni.io"
+  type    = "CNAME"
+  ttl     = 300
+
+  records = [
+    module.ooniapi_citizenlab.aws_instance_public_dns
+  ]
+}
+
+module "citizenlab_builder" {
+  source      = "../../modules/ooni_docker_build"
+  trigger_tag = ""
+
+  service_name            = "citizenlab"
+  repo                    = "ooni/backend"
+  branch_name             = "add_citizenlab_url_management_with_porcelain.3"
+  buildspec_path          = "ooniapi/services/citizenlab/buildspec.yml"
+  trigger_path            = "ooniapi/services/citizenlab/**"
+  codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
+
+  codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
+
+  ecs_cluster_name = module.ooniapi_cluster.cluster_name
+}
+
 #### OONI Tier0 API Frontend
 
 module "ooniapi_frontend" {
@@ -1101,6 +1195,7 @@ module "ooniapi_frontend" {
   ooniapi_ooniprobe_target_group_arn        = module.ooniapi_ooniprobe.alb_target_group_id
   ooniapi_oonifindings_target_group_arn     = module.ooniapi_oonifindings.alb_target_group_id
   ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id
+  ooniapi_citizenlab_target_group_arn       = module.ooniapi_citizenlab.aws_instance_id
 
   ooniapi_service_security_groups = [
     module.ooniapi_cluster.web_security_group_id,
@@ -1127,6 +1222,7 @@ locals {
     "oonirun.${local.environment}.ooni.io" : local.dns_zone_ooni_io,
     "oonimeasurements.${local.environment}.ooni.io" : local.dns_zone_ooni_io,
     "8.th.dev.ooni.io" : local.dns_zone_ooni_io,
+    "citizenlab.${local.environment}.ooni.io" : local.dns_zone_ooni_io,
   }
   ooniapi_frontend_main_domain_name         = "api.${local.environment}.ooni.io"
   ooniapi_frontend_main_domain_name_zone_id = local.dns_zone_ooni_io