diff --git a/ignore_tutorials/ignore_circleci_testing b/ignore_tutorials/ignore_circleci_testing index efac36dc..137218f1 100644 --- a/ignore_tutorials/ignore_circleci_testing +++ b/ignore_tutorials/ignore_circleci_testing @@ -1,13 +1,7 @@ -# ignore these 10 now as excluding execution for these is a combination that passes JB2 building on CircleCI. +# ignore these as they run out of resources on CircleCI. # Note that the first 2 are excluded for all kinds of testing; while the parallelize causes issues for GHA JB2 builds and non linux testing, too. # -tutorials/parquet-catalog-demos/neowise-source-table-lightcurves -tutorials/parquet-catalog-demos/neowise-source-table-strategies -tutorials/parallelize/Parallelize_Convolution tutorials/roman_simulations/roman_hlss_number_density -tutorials/cosmodc2/cosmoDC2_TAP_access tutorials/openuniversesims/openuniverse2024_roman_simulated_timedomainsurvey -tutorials/openuniversesims/openuniverse2024_roman_simulated_wideareasurvey -tutorials/parquet-catalog-demos/wise-allwise-catalog-demo tutorials/cloud_access/euclid-cloud-access tutorials/cloud_access/cloud-access-intro diff --git a/tox.ini b/tox.ini index 17385d00..64f9712b 100644 --- a/tox.ini +++ b/tox.ini @@ -68,24 +68,24 @@ commands = bash -c 'find tutorials -name "*md" | xargs grep kernelspec: | awk -F :kernelspec: "{print \$1}" > all_tutorials' # Make a list of the tutorials changed, we only need this in CI. Also deal with grep's non-zero exit code. - bash -c 'if [[ $CI == true ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep ".md" || true; fi > changed_tutorials' - + #bash -c 'if [[ $CI == true ]]; then git fetch origin main --depth=1; git diff origin/main --name-only tutorials | grep ".md" || true; fi > changed_tutorials' + bash -c 'cat all_tutorials > changed_tutorials' # We only skip testing untouched tutorials in PRs; in cron and dispatch all should be tested !buildhtml: bash -c 'if [[ $GITHUB_EVENT_NAME == pull_request && -z "$(grep force_run:all_tests ${GITHUB_EVENT_PATH})" ]]; then cat changed_tutorials; else cat all_tutorials; fi | grep -vf ignore_testing | xargs jupytext --to notebook ' - !buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:\n'; cat ignore_testing" + !buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:'; cat ignore_testing" !buildhtml: pytest --nbval-lax -vv --suppress-no-test-exit-code --durations=10 tutorials # We don't want to execute in rendering the ones we ignore for testing - buildhtml: bash -c 'grep -f ignore_testing all_tutorials > ignore_execute' + buildhtml: bash -c 'grep -f ignore_testing all_tutorials || true > ignore_execute' # On CircleCI rendering preview, we also don't want to execute anything that hasn't been modified - buildhtml: bash -c 'if [[ $CIRCLECI == true ]]; then grep -vf changed_tutorials all_tutorials >> ignore_execute; fi' + buildhtml: bash -c 'if [[ $CIRCLECI == true ]]; then grep -vf changed_tutorials all_tutorials || true >> ignore_execute; fi' # sed -i needs a bit of hacky conditional on ubuntu to cover the case of an empty ignore buildhtml: bash -c 'if [ -s ignore_execute ]; then for name in $(cat ignore_execute | sort| uniq); do if [ -z "$(head -n 20 ${name}| grep execute:)" ]; then sed -i -e "s|kernelspec:|execute:\n skip: true\nkernelspec:|g" ${name}; fi; done;fi' - buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:\n'; cat ignore_execute" + buildhtml: bash -c "echo 'Notebooks ignored (not tested/executed) in this job:'; cat ignore_execute" # Using srtict so we fail with trackbacks and debug mode to have a richer log # For full build we disable parallel runs to easy server load