diff --git a/.github/workflows/ee-test-with-oauth2.yml b/.github/workflows/ee-test-with-oauth2.yml index 6242b86..79db31b 100644 --- a/.github/workflows/ee-test-with-oauth2.yml +++ b/.github/workflows/ee-test-with-oauth2.yml @@ -19,10 +19,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install earthengine-api + pip install earthengine-api httplib2 - name: Run Earth Engine Script env: + EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }} + EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }} EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_TOKEN }} run: | python ee-test-with-oauth2.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a32158 --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +ENV/ +env/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Earth Engine credentials (for local testing) +.config/ diff --git a/README.md b/README.md index d4e39d0..e00c7cc 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,82 @@ # ee-initialize-github-actions -_Instructions for initializing to Earth Engine in Python scripts run with GitHub Actions. There -are multiple ways to do this, I'd like to add several, but for now it demonstrates constructing -credentials from `google.oauth2.credentials.Credentials`_ +_Instructions for initializing to Earth Engine in Python scripts run with GitHub Actions._ So you want to test Earth Engine in your GitHub Actions? Great idea! To do it, you'll need -to authenticate and initialize to the Earth Engine service. This repo describes how to -generate a credentials file, save those credentials as a GitHub Secret, construct -oauth2 credentials and pass them to `ee.Initialize()`. A basic workflow file and Earth -Engine script are provided, but they are super minimal and not the focus of this demo. +to authenticate and initialize to the Earth Engine service. This repo demonstrates two modern +authentication methods: + +1. **Service Account Authentication** (Recommended for CI/CD) - Uses a Google Cloud service account +2. **Token-based Authentication** (Alternative) - Uses your personal Earth Engine credentials + +Both methods are demonstrated with a basic workflow file and Earth Engine script. + +## Method 1: Service Account Authentication (Recommended) + +This is the **recommended approach for CI/CD workflows** as it's more secure and doesn't rely +on personal credentials. + +### 1. Create a Service Account in Google Cloud + +1. Go to the [Google Cloud Console](https://console.cloud.google.com/) +2. Select your Earth Engine enabled project +3. Navigate to **IAM & Admin > Service Accounts** +4. Click **Create Service Account** +5. Give it a name (e.g., "github-actions-ee") and click **Create** +6. Grant the service account appropriate permissions (at minimum, it needs Earth Engine access) +7. Click **Done** + +### 2. Create a Service Account Key + +1. Click on the service account you just created +2. Go to the **Keys** tab +3. Click **Add Key > Create new key** +4. Select **JSON** format and click **Create** +5. A JSON file will be downloaded - keep this secure! + +### 3. Register the Service Account with Earth Engine + +You need to register this service account with Earth Engine: + +```shell +earthengine set_account @.iam.gserviceaccount.com +``` + +Or register it through the [Earth Engine Code Editor](https://code.earthengine.google.com/) by sharing +assets with the service account email. + +### 4. Add Secrets to GitHub + +1. Go to your GitHub repository +2. Navigate to **Settings > Secrets and variables > Actions** +3. Click **New repository secret** +4. Create two secrets: + - **Name:** `EARTHENGINE_SERVICE_ACCOUNT` + - **Value:** The entire contents of the JSON key file (paste as-is) + - **Name:** `EARTHENGINE_PROJECT` (optional, recommended for clarity) + - **Value:** Your Google Cloud project ID + +Note: For service accounts, the project ID is extracted from the credentials, but setting +`EARTHENGINE_PROJECT` explicitly is recommended for clarity. + +### 5. Update Your Workflow + +Your workflow should set these environment variables: + +```yml +- name: Run Earth Engine Script + env: + EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }} + EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }} + run: | + python ee-test-with-oauth2.py +``` + +--- + +## Method 2: Token-based Authentication (Alternative) + +This method uses your personal Earth Engine credentials. Note that modern Earth Engine credentials +no longer include OAuth2 client credentials. ## 1. Create Earth Engine credentials @@ -39,7 +108,7 @@ One way to do that is to include a default project in your credentials file. Her `earthengine set_project` command. Be sure to edit the project ID to one that you want associated with running tests in your GitHub repo. -To check you existing projects ids you can use the following command +To check your existing projects ids you can use the following command ```shell gcloud projects list @@ -56,71 +125,146 @@ The given project will now appear in the credentials file just created. ## 2. Add the credentials information as a GitHub secret **Find your credentials file (`~/.config/earthengine/credentials`) and open it with a text editor.** -If should be single-line JSON formatted like this: +The modern format should look like this (single-line JSON): ```json -{"client_id": "value", "client_secret": "value", "refresh_token": "value", "project": "value"} +{"refresh_token": "value", "project": "value", ...} ``` -**Go to the GitHub repo where you're running GitHub Actions and create a new -[repository secret](https://docs.github.com/en/codespaces/managing-codespaces-for-your-organization/managing-development-environment-secrets-for-your-repository-or-organization).** +Note: The credentials no longer contain `client_id` and `client_secret` - this is expected! + +**Go to the GitHub repo where you're running GitHub Actions and create repository secrets.** On the top tabs click "Actions", on the left TOC click "Secrets and variables", select "Actions", and then "New repository secret" ![image](https://github.com/user-attachments/assets/65dbd501-fbbd-43dd-b9e0-44d886d7eddb) -You'll be prompted to name the secret; I suggest `EARTHENGINE_TOKEN`. Copy the credentials information -from your text editor into the secret input text box. It's important that the text be unaltered and -unformatted to avoid JSON decoding errors. +Create two secrets: + +1. **Name:** `EARTHENGINE_TOKEN` + - **Value:** Copy the entire credentials file content (keep it as single-line JSON) + +2. **Name:** `EARTHENGINE_PROJECT` (required if not in token) + - **Value:** Your Google Cloud project ID (e.g., "my-ee-project") + +Note: `EARTHENGINE_PROJECT` is required for token-based authentication. If your credentials file +already contains a "project" field, the script will use that value. However, setting the environment +variable explicitly is recommended. > We advise minifying your JSON into a single line string before storing it in a GitHub Secret. When a > GitHub Secret is used in a GitHub Actions workflow, each line of the secret is masked in log output. > This can lead to aggressive sanitization of benign characters like curly braces ({}) and brackets ([]). -## 3. Write your workflow and add the EARTHENGINE_TOKEN secret as an environmental varible +--- + +## 3. Write your workflow and add the secrets as environment variables I'll not get into the [details of writing a workflow](https://docs.github.com/en/actions/writing-workflows/about-workflows). You can [see my full example](https://github.com/gee-community/ee-initialize-github-actions/blob/main/.github/workflows/ee-test-with-oauth2.yml), -but **the important part to note is that I'm setting an environmental variable from the credentials secret -in the step that runs my Earth Engine script** ([ee-test-with-oauth2.py](https://github.com/gee-community/ee-initialize-github-actions/blob/main/ee-test-with-oauth2.py)). +but **the important part is setting the environment variables in the step that runs your Earth Engine script** +([ee-test-with-oauth2.py](https://github.com/gee-community/ee-initialize-github-actions/blob/main/ee-test-with-oauth2.py)). + +**For Service Account Authentication:** + +```yml +- name: Run Earth Engine Script + env: + EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }} + EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }} + run: | + python ee-test-with-oauth2.py +``` + +**For Token-based Authentication:** ```yml - name: Run Earth Engine Script env: EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_TOKEN }} + EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }} run: | python ee-test-with-oauth2.py ``` +The script automatically detects which authentication method to use based on available environment variables. + ## 4. Initialize to Earth Engine in your test file -In the test file ([ee-test-with-oauth2.py](https://github.com/gee-community/ee-initialize-github-actions/blob/main/ee-test-with-oauth2.py)) -that makes Earth Engine requests, **construct Oauth2 credentials -from the credentials info in the secret**. The credentials info is fetched from -the environment variable we set previously, and then arranged as arguments -to `google.oauth2.credentials.Credentials` whose result is given to `ee.Initialize()`. +In the test file ([ee-test-with-oauth2.py](https://github.com/gee-community/ee-initialize-github-actions/blob/main/ee-test-with-oauth2.py)), +the script **automatically detects and uses the appropriate authentication method**: + +1. **Service Account** (if `EARTHENGINE_SERVICE_ACCOUNT` is set) - preferred method +2. **Token-based** (if `EARTHENGINE_TOKEN` is set) - fallback method + +The modern implementation no longer requires manual OAuth2 credential construction: ```python import ee import json import os -import google.oauth2.credentials - -stored = json.loads(os.getenv("EARTHENGINE_TOKEN")) -credentials = google.oauth2.credentials.Credentials( - None, - token_uri="https://oauth2.googleapis.com/token", - client_id=stored["client_id"], - client_secret=stored["client_secret"], - refresh_token=stored["refresh_token"], - quota_project_id=stored["project"], -) - -ee.Initialize(credentials=credentials) +import re +import httplib2 +from pathlib import Path + +def init_ee_from_service_account(): + """Initialize Earth Engine using a service account (preferred for CI/CD).""" + if "EARTHENGINE_SERVICE_ACCOUNT" in os.environ: + private_key = os.environ["EARTHENGINE_SERVICE_ACCOUNT"] + ee_user = json.loads(private_key)["client_email"] + credentials = ee.ServiceAccountCredentials(ee_user, key_data=private_key) + ee.Initialize( + credentials=credentials, + project=credentials.project_id, + http_transport=httplib2.Http() + ) + return True + return False + +def init_ee_from_token(): + """Initialize Earth Engine using a token (fallback method).""" + if "EARTHENGINE_TOKEN" in os.environ: + ee_token = os.environ["EARTHENGINE_TOKEN"] + # Write token to credentials file + credential_folder_path = Path.home() / ".config" / "earthengine" + credential_folder_path.mkdir(parents=True, exist_ok=True) + credential_file_path = credential_folder_path / "credentials" + credential_file_path.write_text(ee_token) + credential_file_path.chmod(0o600) # Set secure permissions + + # Get project ID from environment or token + project_id = os.environ.get("EARTHENGINE_PROJECT") + if project_id is None: + # Try to extract from token + try: + token_data = json.loads(ee_token) + project_id = token_data.get("project") or token_data.get("project_id") + except (json.JSONDecodeError, AttributeError): + pass + + if project_id is None: + raise ValueError( + "Project ID cannot be detected. " + "Please set EARTHENGINE_PROJECT or include 'project' in credentials" + ) + + ee.Initialize(project=project_id, http_transport=httplib2.Http()) + return True + return False + +# Try service account first, then token-based +if not init_ee_from_service_account(): + if not init_ee_from_token(): + raise ValueError("No valid authentication method found") print(ee.String("Greetings from the Earth Engine servers!").getInfo()) ``` +Key changes from the old approach: +- No longer requires `client_id` and `client_secret` +- Supports modern service account authentication +- Works with new credential file format +- Automatic method detection + ## 5. Test the script In this case, I'm just **manually triggering the workflow from the "Actions" tab, diff --git a/ee-test-with-oauth2.py b/ee-test-with-oauth2.py index 5004dba..1249abe 100644 --- a/ee-test-with-oauth2.py +++ b/ee-test-with-oauth2.py @@ -1,19 +1,80 @@ import ee import json import os -import google.oauth2.credentials +import re +import httplib2 +from pathlib import Path -stored = json.loads(os.getenv("EARTHENGINE_TOKEN")) -credentials = google.oauth2.credentials.Credentials( - None, - token_uri="https://oauth2.googleapis.com/token", - client_id=stored["client_id"], - client_secret=stored["client_secret"], - refresh_token=stored["refresh_token"], - quota_project_id=stored["project"], - scopes=["https://www.googleapis.com/auth/earthengine"] -) +def init_ee_from_service_account(): + """Initialize Earth Engine using a service account (preferred for CI/CD).""" + if "EARTHENGINE_SERVICE_ACCOUNT" in os.environ: + try: + private_key = os.environ["EARTHENGINE_SERVICE_ACCOUNT"] + ee_data = json.loads(private_key) + ee_user = ee_data["client_email"] + + # Connect to GEE using a ServiceAccountCredentials object + credentials = ee.ServiceAccountCredentials(ee_user, key_data=private_key) + # httplib2 transport is used for better compatibility in CI/CD environments + ee.Initialize( + credentials=credentials, + project=credentials.project_id, + http_transport=httplib2.Http() + ) + return True + except (json.JSONDecodeError, KeyError) as e: + raise ValueError( + f"Invalid EARTHENGINE_SERVICE_ACCOUNT format: {e}. " + "Please ensure it contains valid service account JSON." + ) + return False -ee.Initialize(credentials=credentials) +def init_ee_from_token(): + """Initialize Earth Engine using a token (fallback method).""" + if "EARTHENGINE_TOKEN" in os.environ: + ee_token = os.environ["EARTHENGINE_TOKEN"] + + # Remove quotes if present (readthedocs workaround) + pattern = re.compile(r"^'[^']*'$") + ee_token = ee_token[1:-1] if pattern.match(ee_token) else ee_token + + # Write the token to the credentials file + credential_folder_path = Path.home() / ".config" / "earthengine" + credential_folder_path.mkdir(parents=True, exist_ok=True) + credential_file_path = credential_folder_path / "credentials" + credential_file_path.write_text(ee_token) + # Set restrictive permissions (owner read/write only) + credential_file_path.chmod(0o600) + + # Get project ID + project_id = os.environ.get("EARTHENGINE_PROJECT") + if project_id is None: + # Try to extract from token + try: + token_data = json.loads(ee_token) + project_id = token_data.get("project") or token_data.get("project_id") + except (json.JSONDecodeError, AttributeError): + pass + + if project_id is None: + raise ValueError( + "Project ID cannot be detected. " + "Please set the EARTHENGINE_PROJECT environment variable or " + "include 'project' field in your credentials." + ) + + # httplib2 transport is used for better compatibility in CI/CD environments + ee.Initialize(project=project_id, http_transport=httplib2.Http()) + return True + return False + +# Try service account authentication first (preferred), then token-based +if not init_ee_from_service_account(): + if not init_ee_from_token(): + raise ValueError( + "No valid authentication method found. " + "Please set either EARTHENGINE_SERVICE_ACCOUNT or EARTHENGINE_TOKEN " + "environment variable." + ) print(ee.String("Greetings from the Earth Engine servers!").getInfo())