From 8e98dfef5b3ad2703da1efd1ec256ba9975fe20d Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 21 May 2025 16:48:56 +0330 Subject: [PATCH 1/2] feat: auto-select region! --- hivemind_etl/storage/s3_client.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/hivemind_etl/storage/s3_client.py b/hivemind_etl/storage/s3_client.py index 343139b..eae8fcc 100644 --- a/hivemind_etl/storage/s3_client.py +++ b/hivemind_etl/storage/s3_client.py @@ -46,18 +46,23 @@ def __init__(self): f"Initializing S3 client with bucket: {self.bucket_name}, region: {self.region}" ) - # Configure S3 client - config = Config( - signature_version="s3v4", - region_name=self.region, + # a region-agnostic client (no region_name) always works for GetBucketLocation + self.s3_client = boto3.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + config=Config(signature_version="s3v4"), ) + resp = self.s3_client.get_bucket_location(Bucket=self.bucket_name) + self.bucket_region = resp["LocationConstraint"] or "us-east-1" + self.s3_client = boto3.client( "s3", - # endpoint_url=self.endpoint_url, + region_name=self.bucket_region, aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, - config=config, + config=Config(signature_version="s3v4"), ) # Ensure bucket exists From 63e6c2c10a27f5d05fdf49918bef59f77882ad3f Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 21 May 2025 16:54:27 +0330 Subject: [PATCH 2/2] feat: logging the region! --- hivemind_etl/storage/s3_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hivemind_etl/storage/s3_client.py b/hivemind_etl/storage/s3_client.py index eae8fcc..12fc59d 100644 --- a/hivemind_etl/storage/s3_client.py +++ b/hivemind_etl/storage/s3_client.py @@ -57,6 +57,8 @@ def __init__(self): resp = self.s3_client.get_bucket_location(Bucket=self.bucket_name) self.bucket_region = resp["LocationConstraint"] or "us-east-1" + logging.info(f"Bucket region: {self.bucket_region}!") + self.s3_client = boto3.client( "s3", region_name=self.bucket_region,