From 41dad5d25eafe230f5a2cc2c488e31df882816d7 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 27 Jan 2026 13:14:05 -0600 Subject: [PATCH 1/3] fix: route logging to stdout, simplify format, add JOBS level - Route all DataJoint logs to stdout instead of stderr (avoids red highlighting in terminals/IDEs that color stderr) - Simplify INFO/DEBUG format: remove level tag for cleaner output - Keep [WARNING] and [ERROR] tags for actual issues - Add JOBS log level (15, between DEBUG and INFO) for make status messages (start/error/success) so they don't flood INFO logs Log levels: DEBUG (10) < JOBS (15) < INFO (20) < WARNING (30) < ERROR (40) Example output: [2026-01-27 10:30:45] DataJoint 2.0.0 connected to postgres@localhost:5432 [2026-01-27 10:30:46][JOBS]: Making {'key': 1} -> schema.Table [2026-01-27 10:30:47][JOBS]: Success making {'key': 1} -> schema.Table [2026-01-27 10:31:02][WARNING]: Table xyz not found Users can filter job messages with DJ_LOG_LEVEL=INFO (hides JOBS) or see them with DJ_LOG_LEVEL=JOBS or DJ_LOG_LEVEL=DEBUG. Co-Authored-By: Claude Opus 4.5 --- src/datajoint/autopopulate.py | 6 +++--- src/datajoint/logging.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/datajoint/autopopulate.py b/src/datajoint/autopopulate.py index b40ebbda4..c02f18791 100644 --- a/src/datajoint/autopopulate.py +++ b/src/datajoint/autopopulate.py @@ -598,7 +598,7 @@ def _populate1( jobs.complete(key) return False - logger.debug(f"Making {key} -> {self.full_table_name}") + logger.jobs(f"Making {key} -> {self.full_table_name}") self.__class__._allow_insert = True try: @@ -629,7 +629,7 @@ def _populate1( exception=error.__class__.__name__, msg=": " + str(error) if str(error) else "", ) - logger.debug(f"Error making {key} -> {self.full_table_name} - {error_message}") + logger.jobs(f"Error making {key} -> {self.full_table_name} - {error_message}") if jobs is not None: jobs.error(key, error_message=error_message, error_stack=traceback.format_exc()) if not suppress_errors or isinstance(error, SystemExit): @@ -640,7 +640,7 @@ def _populate1( else: self.connection.commit_transaction() duration = time.time() - start_time - logger.debug(f"Success making {key} -> {self.full_table_name}") + logger.jobs(f"Success making {key} -> {self.full_table_name}") # Update hidden job metadata if table has the columns if self._has_job_metadata_attrs(): diff --git a/src/datajoint/logging.py b/src/datajoint/logging.py index b432e1a4b..a87ecfc74 100644 --- a/src/datajoint/logging.py +++ b/src/datajoint/logging.py @@ -2,14 +2,42 @@ import os import sys +# Custom log level for job/populate status messages +# DEBUG (10) < JOBS (15) < INFO (20) < WARNING (30) < ERROR (40) +JOBS = 15 +logging.addLevelName(JOBS, "JOBS") + + +def jobs(self, message, *args, **kwargs): + """Log job status messages (make start/success/error).""" + if self.isEnabledFor(JOBS): + self._log(JOBS, message, args, **kwargs) + + +logging.Logger.jobs = jobs + logger = logging.getLogger(__name__.split(".")[0]) log_level = os.getenv("DJ_LOG_LEVEL", "info").upper() -log_format = logging.Formatter("[%(asctime)s][%(levelname)s]: %(message)s") -stream_handler = logging.StreamHandler() # default handler -stream_handler.setFormatter(log_format) +class LevelAwareFormatter(logging.Formatter): + """Format INFO messages cleanly, show level for warnings/errors and JOBS.""" + + def format(self, record): + timestamp = self.formatTime(record, "%Y-%m-%d %H:%M:%S") + if record.levelno >= logging.WARNING: + return f"[{timestamp}][{record.levelname}]: {record.getMessage()}" + elif record.levelno == JOBS: + return f"[{timestamp}][JOBS]: {record.getMessage()}" + else: + return f"[{timestamp}] {record.getMessage()}" + + +# Use stdout for all logging (avoids red highlighting in terminals/IDEs) +# Users needing stderr can configure their own handlers +stream_handler = logging.StreamHandler(sys.stdout) +stream_handler.setFormatter(LevelAwareFormatter()) logger.setLevel(level=log_level) logger.handlers = [stream_handler] From 87cd5b048e6bba5c7b52e37ae0c3c6e54bea6826 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 27 Jan 2026 14:17:12 -0600 Subject: [PATCH 2/3] chore: bump version to 2.0.0a26 --- src/datajoint/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datajoint/version.py b/src/datajoint/version.py index 068e6fe5c..acc17bb66 100644 --- a/src/datajoint/version.py +++ b/src/datajoint/version.py @@ -1,4 +1,4 @@ # version bump auto managed by Github Actions: # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit) # manually set this version will be eventually overwritten by the above actions -__version__ = "2.0.0a25" +__version__ = "2.0.0a26" From 2722a4d24b4f385dabca79db244b57ef0441107a Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 27 Jan 2026 14:20:50 -0600 Subject: [PATCH 3/3] feat: add DJ_LOG_STREAM env var to select stdout/stderr --- src/datajoint/logging.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/datajoint/logging.py b/src/datajoint/logging.py index a87ecfc74..280d08167 100644 --- a/src/datajoint/logging.py +++ b/src/datajoint/logging.py @@ -19,6 +19,7 @@ def jobs(self, message, *args, **kwargs): logger = logging.getLogger(__name__.split(".")[0]) log_level = os.getenv("DJ_LOG_LEVEL", "info").upper() +log_stream = os.getenv("DJ_LOG_STREAM", "stdout").lower() class LevelAwareFormatter(logging.Formatter): @@ -34,9 +35,10 @@ def format(self, record): return f"[{timestamp}] {record.getMessage()}" -# Use stdout for all logging (avoids red highlighting in terminals/IDEs) -# Users needing stderr can configure their own handlers -stream_handler = logging.StreamHandler(sys.stdout) +# Select output stream: stdout (default, no red highlighting) or stderr +# Configurable via DJ_LOG_STREAM=stdout|stderr +output_stream = sys.stderr if log_stream == "stderr" else sys.stdout +stream_handler = logging.StreamHandler(output_stream) stream_handler.setFormatter(LevelAwareFormatter()) logger.setLevel(level=log_level)