From d380fa67f68d1dfddc006101bc299aa2d2518141 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 19 Jul 2026 13:35:47 +0100 Subject: [PATCH 01/25] Refactor the "Standard Constants" section. Rename the "Standard Constants" section to "Common variables" (those variables do not follow any kind of "standard"). Remove some variables that are of dubious use, such as `FORMATS_INCL_TSV` (used only once). Move some variables into the section in which they are actually used. Only keep in the "Common variables" section the variables that are actually common throughout the entire Makefile. --- .../templates/src/ontology/Makefile.jinja2 | 189 ++++++++++-------- 1 file changed, 101 insertions(+), 88 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index c41c287..d06ec8a 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -23,100 +23,97 @@ CONFIG_HASH= {{ project.config_hash }} {% endif %} # ---------------------------------------- -# Standard Constants -# ---------------------------------------- -# these can be overwritten on the command line - -OBOBASE= http://purl.obolibrary.org/obo -URIBASE= {{ project.uribase }} -ONT= {{ project.id }} -ONTBASE= {{ project.uribase }}/{% if project.uribase_suffix is not none %}{{ project.uribase_suffix }}{% else %}{{ project.id }}{% endif %} -EDIT_FORMAT= {{ project.edit_format|default('owl') }} -SRC = $(ONT)-edit.$(EDIT_FORMAT) -MAKE_FAST= $(MAKE) IMP=false PAT=false COMP=false MIR=false -CATALOG= {{ project.catalog_file }} +# Common variables +# ---------------------------------------- +# These variables are intended to be used throughout the Makefile (and +# the custom {{ project.id }}.Makefile, if needed). They _may_ be +# overriden either on the command line or in the custom Makefile, but in +# most cases this is not recommended. + +# Project settings +OBOBASE = http://purl.obolibrary.org/obo +URIBASE = {{ project.uribase }} +ONT = {{ project.id }} +ONTBASE = {{ project.uribase }}/{% if project.uribase_suffix is not none %}{{ project.uribase_suffix }}{% else %}{{ project.id }}{% endif %} +EDIT_FORMAT = {{ project.edit_format|default('owl') }} +FORMATS = $(sort {% for format in project.export_formats %} {{ format }}{% endfor %} owl) +RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}) + +# Directories +RELEASEDIR = ../.. +DOCSDIR = ../../docs +REPORTDIR = reports +TMPDIR = tmp +MIRRORDIR = mirror +IMPORTDIR = imports +SUBSETDIR = subsets +SCRIPTSDIR = ../scripts +UPDATEREPODIR = target +SPARQLDIR = ../sparql +COMPONENTSDIR = components +TEMPLATEDIR = ../templates +{% if project.use_dosdps -%} +PATTERNDIR = ../patterns +{% endif -%} +{% if project.use_mappings -%} +MAPPINGDIR = ../mappings +{% endif -%} +{% if project.bridge_group is not none -%} +BRIDGEDIR = bridges +{% endif -%} +{% if project.use_translations -%} +TRANSLATIONSDIR= = ../translations +{% endif -%} +ODK_RESOURCES_DIR ?= /tools/resources + +# Files +SRC = $(ONT)-edit.$(EDIT_FORMAT) +OTHER_SRC = {% if project.use_dosdps %}$(PATTERNDIR)/definitions.owl {% endif %}{% if project.components is defined %}{% for component in project.components.products %}$(COMPONENTSDIR)/{{ component.filename }} {% endfor %}{% endif %} +CATALOG = {{ project.catalog_file }} {% if project.use_context -%} -CONTEXT_FILE = config/context.json +CONTEXT_FILE = config/context.json {% endif -%} -ROBOT= robot --catalog $(CATALOG){% if project.use_context %} --add-prefixes $(CONTEXT_FILE){% endif %} -REASONER= {{ project.robot.reasoner }} -{# Kept for backwards compatibility with existing custom Makefiles -#} +EXTENDED_PREFIX_MAP = $(TMPDIR)/obo.epm.json +ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt +EDIT_PREPROCESSED = $(TMPDIR)/$(ONT)-preprocess.owl +SRCMERGED = $(TMPDIR)/merged-$(ONT)-edit.ofn + +# ROBOT Options +ROBOT = robot --catalog $(CATALOG){% if project.use_context %} --add-prefixes $(CONTEXT_FILE){% endif %} +REASONER = {{ project.robot.reasoner }} +REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} +REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}--labels true{% endif %} +REPORT_PROFILE_OPTS = {% if project.robot.report.custom_profile %}--profile profile.txt{% endif %} +OBO_FORMAT_OPTIONS = {{ project.robot.obo_format_options }} +RELAX_OPTIONS = {{ project.robot.relax_options }} +REDUCE_OPTIONS = {{ project.robot.reduce_options }} + +{# Kept for backward compatibility with existing custom Makefiles -#} +# OWLTools Options {% if project.owltools_memory|length -%} -OWLTOOLS_MEMORY = {{ project.owltools_memory }} +OWLTOOLS_MEMORY = {{ project.owltools_memory }} {% endif -%} -OWLTOOLS = {% if project.owltools_memory|length %}OWLTOOLS_MEMORY=$(OWLTOOLS_MEMORY) {% endif %}owltools --use-catalog -RELEASEDIR= ../.. -DOCSDIR= ../../docs -REPORTDIR= reports -TEMPLATEDIR= ../templates -TMPDIR= tmp -MIRRORDIR= mirror -IMPORTDIR= imports -SUBSETDIR= subsets -SCRIPTSDIR= ../scripts -UPDATEREPODIR= target -SPARQLDIR = ../sparql -EXTENDED_PREFIX_MAP= $(TMPDIR)/obo.epm.json -COMPONENTSDIR = components -ODK_RESOURCES_DIR ?= /tools/resources -{%- if project.robot.report.custom_profile %} -ROBOT_PROFILE = profile.txt -{%- endif %} -REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} -REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}-l true{% endif %} -REPORT_PROFILE_OPTS = {% if project.robot.report.custom_profile %}--profile $(ROBOT_PROFILE){% endif %} -OBO_FORMAT_OPTIONS = {{ project.robot.obo_format_options }} -SPARQL_VALIDATION_CHECKS = {% for x in project.robot.report.custom_sparql_checks|default(['owldef-self-reference', 'iri-range', 'label-with-iri', 'multiple-replaced_by']) %}{{ x }} {% endfor %} -SPARQL_EXPORTS = {% for x in project.robot.report.custom_sparql_exports|default(['basic-report', 'class-count-by-prefix', 'edges', 'xrefs', 'obsoletes', 'synonyms']) %}{{ x }} {% endfor %} -ODK_VERSION_MAKEFILE = {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %} -RELAX_OPTIONS = {{ project.robot.relax_options }} -REDUCE_OPTIONS = {{ project.robot.reduce_options }} - -TODAY ?= $(shell date +%Y-%m-%d) -OBODATE ?= $(shell date +'%d:%m:%Y %H:%M') -VERSION= $(TODAY) -ANNOTATE_ONTOLOGY_VERSION = annotate -V $(ONTBASE)/releases/$(VERSION)/$@ --annotation owl:versionInfo $(VERSION) -ANNOTATE_CONVERT_FILE = annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) convert -f {{ project.import_component_format|default('ofn') }} --output $@.tmp.owl && mv $@.tmp.owl $@ -OTHER_SRC = {% if project.use_dosdps -%}$(PATTERNDIR)/definitions.owl {% endif -%}{% if project.components is defined -%}{% for component in project.components.products -%}$(COMPONENTSDIR)/{{ component.filename }} {% endfor -%}{% endif %} -ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt -EDIT_PREPROCESSED = $(TMPDIR)/$(ONT)-preprocess.owl -{% if project.use_context and 'db' in project.export_formats -%} -CONTEXT_FILE_CSV = $(TMPDIR)/context.csv -{%- endif -%} - -{%- if project.use_dosdps %} -PATTERNDIR= ../patterns -PATTERN_TESTER= dosdp validate -i -DOSDPT= dosdp-tools -PATTERN_RELEASE_FILES= $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl -{% endif %} +OWLTOOLS = {% if project.owltools_memory|length %}OWLTOOLS_MEMORY=$(OWLTOOLS_MEMORY) {% endif %}owltools --use-catalog -{%- if project.use_mappings %} -MAPPINGDIR= ../mappings -MAPPING_TESTER= sssom validate -SSSOMPY= sssom -MAPPINGS= {% if project.sssom_mappingset_group is not none %}{%- for mapping in project.sssom_mappingset_group.products %}{{ mapping.id }} {% endfor %}{% endif %} -MAPPING_RELEASE_FILES= $(foreach n,$(MAPPINGS), $(MAPPINGDIR)/$(n).sssom.tsv) -{% endif %} -{% if project.bridge_group is not none -%} -BRIDGEDIR= bridges +# Date & version +TODAY ?= $(shell date +%Y-%m-%d) +{% if project.release_date -%} +OBODATE ?= $(shell date +'%d:%m:%Y %H:%M') {% endif -%} +VERSION = $(TODAY) -{%- if project.use_translations %} -TRANSLATIONSDIR= ../translations -BABELONPY= babelon -q -TRANSLATIONS_OWL= {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.owl {% if translation.include_robot_template_synonyms %} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.owl {% endif %}{% endfor %} -TRANSLATIONS_TSV= {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv {% endfor %} -TRANSLATION_FILES= {%- if project.babelon_translation_group is not none %}{% if project.babelon_translation_group.release_merged_translations %}$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json{% endif %}{% endif %} -{% endif %} - -FORMATS = $(sort {% for format in project.export_formats %} {{ format }}{% endfor %} owl) -FORMATS_INCL_TSV = $(sort $(FORMATS) tsv) -RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}) +# Commands +MAKE_FAST = $(MAKE) IMP=false PAT=false COMP=false MIR=false +ANNOTATE_ONTOLOGY_VERSION = annotate --version-iri $(ONTBASE)/releases/$(VERSION)/$@ \ + --annotation owl:versionInfo $(VERSION) +ANNOTATE_CONVERT_FILE = $(ANNOTATE_ONTOLOGY_VERSION) --ontology-iri $(ONTBASE)/$@ \ + convert --format {{ project.import_component_format|default('ofn') }} \ + --output $@ +# Debugging ifeq ($(ODK_DEBUG),yes) -ODK_DEBUG_FILE = debug.log -SHELL = $(SCRIPTSDIR)/run-command.sh +ODK_DEBUG_FILE = debug.log +SHELL = $(SCRIPTSDIR)/run-command.sh endif # ---------------------------------------- @@ -172,7 +169,7 @@ reason_test: $(EDIT_PREPROCESSED) .PHONY: odkversion odkversion: - @echo "ODK Makefile $(ODK_VERSION_MAKEFILE)" + @echo "ODK Makefile {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %}" @odk-helper info --tools {%- if project.config_hash %} @@ -235,7 +232,6 @@ $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar: MAIN_PRODUCTS = $(sort $(foreach r,$(RELEASE_ARTEFACTS), $(r)) $(ONT)) MAIN_GZIPPED = {% if project.gzip_main %}$(foreach f,$(FORMATS), $(ONT).$(f).gz){% endif %} MAIN_FILES = $(foreach n,$(MAIN_PRODUCTS), $(foreach f,$(FORMATS), $(n).$(f))) $(MAIN_GZIPPED) -SRCMERGED = $(TMPDIR)/merged-$(ONT)-edit.ofn .PHONY: all_main all_main: $(MAIN_FILES) @@ -272,7 +268,7 @@ SUBSETS = {% for x in project.subset_group.products %} {{ x.id }}{% endfor %} SUBSETS = {% endif %} SUBSET_ROOTS = $(patsubst %, $(SUBSETDIR)/%, $(SUBSETS)) -SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(foreach f,$(FORMATS_INCL_TSV), $(n).$(f))) +SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(foreach f,$(sort $(FORMATS) tsv), $(n).$(f))) .PHONY: all_subsets all_subsets: $(SUBSET_FILES) @@ -344,6 +340,7 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt # these live in the ../sparql directory, and have suffix -violation.sparql # adding the name here will make the violation check live. +SPARQL_VALIDATION_CHECKS = {% for x in project.robot.report.custom_sparql_checks|default(['owldef-self-reference', 'iri-range', 'label-with-iri', 'multiple-replaced_by']) %}{{ x }} {% endfor %} SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql) sparql_test: {% for x in project.robot.report.sparql_test_on|default(["edit"]) %} {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %}{% endfor %} | $(REPORTDIR) @@ -1003,6 +1000,8 @@ $(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl # Sparql queries: Table exports / Query Reports # --------------------------------------------- +SPARQL_EXPORTS = {% for x in project.robot.report.custom_sparql_exports|default(['basic-report', 'class-count-by-prefix', 'edges', 'xrefs', 'obsoletes', 'synonyms']) %}{{ x }} {% endfor %} + SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql $(REPORTDIR)/$(V).tsv) # This combines all into one single command @@ -1017,6 +1016,10 @@ endif # DOSDP Templates/Patterns # ---------------------------------------- +PATTERN_TESTER = dosdp validate -i +DOSDPT = dosdp-tools +PATTERN_RELEASE_FILES = $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl + ALL_PATTERN_FILES=$(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml) ALL_PATTERN_NAMES=$(strip $(patsubst %.yaml,%, $(notdir $(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml)))) @@ -1171,6 +1174,9 @@ endif # SSSOM Mapping Files # ---------------------------------------- +MAPPING_TESTER = sssom validate +SSSOMPY = sssom + validate-sssom-%: tsvalid $(MAPPINGDIR)/$*.sssom.tsv --comment "#" $(MAPPING_TESTER) $(MAPPINGDIR)/$*.sssom.tsv @@ -1263,6 +1269,11 @@ $(BRIDGEDIR)/{{ bridge.id }}.owl: # Babelon Translation Files # ---------------------------------------- +BABELONPY = babelon -q +TRANSLATIONS_OWL = {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.owl {% if translation.include_robot_template_synonyms %} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.owl {% endif %}{% endfor %} +TRANSLATIONS_TSV = {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv {% endfor %} +TRANSLATION_FILES = {%- if project.babelon_translation_group is not none %}{% if project.babelon_translation_group.release_merged_translations %}$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json{% endif %}{% endif %} + {%- if project.babelon_translation_group is not none %} TRANSLATIONS_ADAPTER={{ project.babelon_translation_group.oak_adapter|default('pronto:$(ONT).obo') }} @@ -1361,7 +1372,9 @@ $(TRANSLATIONSDIR)/%.babelon.json: $(TRANSLATIONSDIR)/%.babelon.tsv {% endfor -%} {% if 'db' in project.export_formats -%} + {% if project.use_context -%} +CONTEXT_FILE_CSV = $(TMPDIR)/context.csv $(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) @odk-helper context2csv $< > $@ {% endif -%} From 1c6e01b2229c24dccb9d5b988f9c251adad20a27 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 26 Jul 2026 17:16:18 +0100 Subject: [PATCH 02/25] Remove the UPDATEREPODIR variable. Since ODK 1.6, the mechanism to update a ODK-managed repository no longer involves a temporary `target` directory. The repo is updated "in place". --- src/incatools/odk/templates/src/ontology/Makefile.jinja2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index d06ec8a..8deee9f 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -48,7 +48,6 @@ MIRRORDIR = mirror IMPORTDIR = imports SUBSETDIR = subsets SCRIPTSDIR = ../scripts -UPDATEREPODIR = target SPARQLDIR = ../sparql COMPONENTSDIR = components TEMPLATEDIR = ../templates @@ -1640,7 +1639,7 @@ update_docs: .PHONY: clean clean:{% if project.use_dosdps %} $(MAKE) pattern_clean{%- endif %} - for dir in $(MIRRORDIR) $(TMPDIR) $(UPDATEREPODIR) ; do \ + for dir in $(MIRRORDIR) $(TMPDIR) ; do \ reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \ case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \ done From b2e218c591b583ca04daedddd29ad1fdf856f5c9 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 9 Aug 2026 19:32:29 +0100 Subject: [PATCH 03/25] Reformat the beginning of the Makefile template. Reformat the beginning of the Makefile template, from the header to the section about ROBOT plugins (included). Reformatting includes: * changing the order of the "top-level targets" so that they match the order in which they are declared in the `all_odk` rule, for better readability; * fixing Jinja2 whitespace handling. --- .../templates/src/ontology/Makefile.jinja2 | 82 +++++++++++-------- 1 file changed, 48 insertions(+), 34 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 8deee9f..42ba7cd 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1,27 +1,26 @@ {#- - Jinja2 Template for Makefile This file is a template that generates the Makefile. This comment will not be included. - See: http://jinja.pocoo.org/docs/2.10/templates + See: https://jinja.palletsprojects.com/en/stable/templates/ -#} # ---------------------------------------- # Makefile for {{ project.id }} -# Generated using ontology-development-kit +# Generated using the Ontology Development Kit (ODK) # ODK Version: {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %} # ---------------------------------------- # IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use {{ project.id }}.Makefile instead {{ project.custom_makefile_header }} -{%- if project.config_hash %} +{% if project.config_hash -%} # Fingerprint of the configuration file when this Makefile was last generated CONFIG_HASH= {{ project.config_hash }} -{% endif %} +{% endif -%} # ---------------------------------------- # Common variables # ---------------------------------------- @@ -115,6 +114,7 @@ ODK_DEBUG_FILE = debug.log SHELL = $(SCRIPTSDIR)/run-command.sh endif + # ---------------------------------------- # Workflow control # ---------------------------------------- @@ -138,6 +138,7 @@ COMP = true # Re-generation of pattern-derived files PAT = true + # ---------------------------------------- # Top-level targets # ---------------------------------------- @@ -150,39 +151,46 @@ all: all_odk .PHONY: all_odk all_odk: odkversion{% if project.config_hash %} config_check{% endif %} test custom_reports all_assets{% if project.release_diff %} release_diff{% endif %} -.PHONY: test -test: odkversion validate_idranges {% if project.use_dosdps %}dosdp_validation {% endif %}reason_test sparql_test robot_reports {% if project.robot.report.ensure_owl2dl_profile|default(true) %}$(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{% endif %} - echo "Finished running all tests successfully." - -.PHONY: test -test_fast: - $(MAKE_FAST) test - -.PHONY: release_diff -release_diff: $(REPORTDIR)/release-diff.md - -.PHONY: reason_test -reason_test: $(EDIT_PREPROCESSED) - $(ROBOT) reason --input $< --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} \ - --exclude-tautologies {{ project.exclude_tautologies }} --output test.owl && rm test.owl - .PHONY: odkversion odkversion: @echo "ODK Makefile {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %}" @odk-helper info --tools -{%- if project.config_hash %} +{% if project.config_hash -%} .PHONY: config_check config_check: @if [ "$$(tr -d '\r' < $(ONT)-odk.yaml | sha256sum | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \ echo "Repository is up-to-date." ; else \ - echo "Your ODK configuration has changed since this Makefile was generated. You may need to run 'sh run.sh update_repo'." ; fi -{% endif %} + echo "Your ODK configuration has changed since this Makefile was generated." ; \ + echo "You may need to run 'sh run.sh update_repo'." ; \ + fi + +{% endif -%} + +.PHONY: test +test: odkversion validate_idranges{% if project.use_dosdps %} dosdp_validation{% endif %} reason_test sparql_test robot_reports{% if project.robot.report.ensure_owl2dl_profile|default(true) %} \ + $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{% endif %} + @echo "Finished running all tests successfully." + +.PHONY: test_fast +test_fast: + $(MAKE_FAST) test + +.PHONY: reason_test +reason_test: $(EDIT_PREPROCESSED) + $(ROBOT) reason --input $< \ + --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} + +.PHONY: release_diff +release_diff: $(REPORTDIR)/release-diff.md $(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR) {%- if project.bridge_group is not none %} $(BRIDGEDIR){% endif %}: mkdir -p $@ + # ---------------------------------------- # ODK-managed ROBOT plugins # ---------------------------------------- @@ -194,10 +202,11 @@ export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins .PHONY: custom_robot_plugins custom_robot_plugins: -{% if project.robot.plugins is not none %} +{% if project.robot.plugins is not none -%} .PHONY: extra_robot_plugins extra_robot_plugins: {% for plugin in project.robot.plugins %} $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar {% endfor %} -{% endif %} + +{% endif -%} # Install all ROBOT plugins to the runtime plugins directory .PHONY: all_robot_plugins @@ -214,16 +223,21 @@ $(ROBOT_PLUGINS_DIRECTORY)/%.jar: cp $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar $@ ; \ fi -# Specific rules for supplementary plugins defined in configuration -{% if project.robot.plugins is not none %}{% for plugin in project.robot.plugins %} +{% if project.robot.plugins is not none -%} +# Additional rules for plugins defined in the ODK configuration +{% for plugin in project.robot.plugins -%} $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar: -{%- if plugin.mirror_from %} - curl -L -o $@ {{ plugin.mirror_from }} -{%- else %} - echo "ERROR: No URL has been provided for this plugin; you must install it yourself by overwriting this rule in {{ project.id }}.Makefile!" && false -{% endif %} -{% endfor %}{% endif %} +{% if plugin.mirror_from -%} +{# #} curl -L -o $@ {{ plugin.mirror_from }} +{% else -%} +{# #} @echo "ERROR: No URL has been provided for this plugin; you must override" + @echo " this rule in {{ project.id }}.Makefile to install it." + @false +{% endif -%} +{# #} +{% endfor -%} +{% endif %} # ---------------------------------------- # Release assets # ---------------------------------------- From e48740d8ce5e6293711d842061fcd5fc465b5d65 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 9 Aug 2026 21:18:47 +0100 Subject: [PATCH 04/25] Move and reformat the QC section. Move the "Quality control" section near the beginning of the Makefile and reformat it. --- .../templates/src/ontology/Makefile.jinja2 | 186 +++++++++--------- 1 file changed, 95 insertions(+), 91 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 42ba7cd..71757d8 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -79,9 +79,6 @@ SRCMERGED = $(TMPDIR)/merged-$(ONT)-edit.ofn # ROBOT Options ROBOT = robot --catalog $(CATALOG){% if project.use_context %} --add-prefixes $(CONTEXT_FILE){% endif %} REASONER = {{ project.robot.reasoner }} -REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} -REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}--labels true{% endif %} -REPORT_PROFILE_OPTS = {% if project.robot.report.custom_profile %}--profile profile.txt{% endif %} OBO_FORMAT_OPTIONS = {{ project.robot.obo_format_options }} RELAX_OPTIONS = {{ project.robot.relax_options }} REDUCE_OPTIONS = {{ project.robot.reduce_options }} @@ -238,6 +235,101 @@ $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar: {% endfor -%} {% endif %} +# ---------------------------------------- +# Quality control +# ---------------------------------------- + +# ROBOT reports +# ---------------------------------------- + +OBO_REPORT ={% for x in project.robot.report.report_on|default(["edit"]) -%} + {% if x=="edit" %} $(SRC){% else %} {{ x }}{% endif %}-obo-report{% endfor %} +{% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} +ALIGNMENT_REPORT ={% for x in project.robot.report.report_on|default(["edit"]) -%} + {% if x=="edit" %} $(SRC){% else %} {{ x }}{% endif %}-align-report{% endfor %} +{% endif -%} +REPORTS = $(OBO_REPORT){% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} + {# #} $(ALIGNMENT_REPORT){% endif %} +REPORT_FILES = $(foreach report, $(REPORTS), $(REPORTDIR)/$(report).tsv) +REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} +REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}--labels true{% endif %} +REPORT_PROFILE_OPTS = {% if project.robot.report.custom_profile %}--profile profile.txt{% endif %} + +.PHONY: robot_reports +robot_reports: $(REPORT_FILES) + +.PHONY: all_reports +all_reports: custom_reports robot_reports + +$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) + $(ROBOT) report --input $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) \ + --fail-on $(REPORT_FAIL_ON) \ + {% if project.robot.report.use_base_iris -%} + {% if project.namespaces is not none -%} + {% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor -%} + {% else -%}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ + --base-iri $(URIBASE)/{{ project.id }} \ + {% endif -%} + {% endif -%} + --print 5 --output $@ + +{% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} +$(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins + $(ROBOT) odk:check-align --input $< --reasoner $(REASONER) \ + --upper-ontology-iri {{ project.robot.report.upper_ontology }} \ + {% if project.robot.report.use_base_iris %}--detail BASE-ROOT \ + {% if project.namespaces is not none -%} + {% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor -%} + {% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ + --base-iri $(URIBASE)/{{ project.id }} \ + {% endif -%} + {% endif -%} + --report-output $@ + +{% endif -%} + +{% if project.robot.report.custom_profile -%} +check_for_robot_updates: + @odk-helper check-robot-profile + +{% endif -%} + +# ROBOT OWL profile check +# ---------------------------------------- + +# The merge step is necessary to avoid undeclared entity violations. +$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) + $(ROBOT) merge --input $< \ + validate-profile --profile DL \ + --output $@ +.PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt + +validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt + @echo "$* profile validation completed." + +# SPARQL-based checks +# ---------------------------------------- + +# The SPARQL queries used by those checks live in the ../sparql directory, +# in files whose name is of the form NAME-violation.sparql. Adding the +# NAME in the list below will enable the corresponding check. + +{# FIXME: The default list should be computed in the seeding script, not here. -#} +SPARQL_VALIDATION_CHECKS ={% for x in project.robot.report.custom_sparql_checks|default(["owldef-self-reference", "iri-range", "label-with-iri", "multiple-replaced_by"]) %} {{ x }}{% endfor %} +SPARQL_VALIDATION_QUERIES = $(foreach name, $(SPARQL_VALIDATION_CHECKS), $(SPARQLDIR)/$(name)-violation.sparql) + +sparql_test:{% for x in project.robot.report.sparql_test_on|default(["edit"]) %} {% if x =="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %}{% endfor %} | $(REPORTDIR) +ifneq ($(SPARQL_VALIDATION_QUERIES),) +{% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} +{# #} $(ROBOT) verify --input {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %} \ + --queries $(SPARQL_VALIDATION_QUERIES) \ + --output-dir $(REPORTDIR) +{% endfor -%} +endif + + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -318,94 +410,6 @@ all_bridges: $(BRIDGE_FILES) BRIDGE_FILES = {% endif -%} -# ---------------------------------------- -# QC Reports & Utilities -# ---------------------------------------- - -OBO_REPORT = {% for x in project.robot.report.report_on|default(["edit"]) %} {% if x=="edit" %}$(SRC){% else %}{{ x }}{% endif %}-obo-report{% endfor %} -ALIGNMENT_REPORT = {% for x in project.robot.report.report_on|default(["edit"]) %} {% if x =="edit" %}$(SRC){% else %}{{ x }}{% endif %}-align-report{% endfor %} -REPORTS = $(OBO_REPORT){% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology %} $(ALIGNMENT_REPORT){% endif %} -REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) - -.PHONY: robot_reports -robot_reports: $(REPORT_FILES) - -.PHONY: all_reports -all_reports: custom_reports robot_reports - -# ---------------------------------------- -# ROBOT OWL Profile checking -# ---------------------------------------- - -# The merge step is necessary to avoid undeclared entity violations. -$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) $(TMPDIR) - $(ROBOT) merge -i $< convert -f ofn -o $(TMPDIR)/validate.ofn - $(ROBOT) validate-profile --profile DL -i $(TMPDIR)/validate.ofn -o $@ || { cat $@ && exit 1; } -.PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt - -validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt - echo "$* profile validation completed." - -# ---------------------------------------- -# Sparql queries: Q/C -# ---------------------------------------- - -# these live in the ../sparql directory, and have suffix -violation.sparql -# adding the name here will make the violation check live. - -SPARQL_VALIDATION_CHECKS = {% for x in project.robot.report.custom_sparql_checks|default(['owldef-self-reference', 'iri-range', 'label-with-iri', 'multiple-replaced_by']) %}{{ x }} {% endfor %} -SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql) - -sparql_test: {% for x in project.robot.report.sparql_test_on|default(["edit"]) %} {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %}{% endfor %} | $(REPORTDIR) -ifneq ($(SPARQL_VALIDATION_QUERIES),) - {% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} - {%- if x=="edit" -%} - {% set input = "$(SRCMERGED)"%} - {%- else -%} - {% set input = x %} - {% endif %} - $(ROBOT) verify -i {{ input }} --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR) - {%- endfor %} -endif - -# ---------------------------------------- -# ROBOT report -# ---------------------------------------- - -$(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot.report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor -%}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif -%}{% endif -%} --print 5 -o $@ - -$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot.report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif -%} --print 5 -o $@ -{%- if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology %} - -$(REPORTDIR)/$(SRC)-align-report.tsv: $(SRCMERGED) | $(REPORTDIR) all_robot_plugins - $(ROBOT) odk:check-align -i $< --reasoner $(REASONER) \ - --upper-ontology-iri {{ project.robot.report.upper_ontology }} - {%- if project.robot.report.use_base_iris %} \ - --detail BASE-ROOT \ - {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} \ - {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} \ - {% endif %}{% else %} \ - {% endif %}--report-output $@ - -$(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins - $(ROBOT) odk:check-align -i $< --reasoner $(REASONER) \ - --upper-ontology-iri {{ project.robot.report.upper_ontology }} - {%- if project.robot.report.use_base_iris %} \ - --detail BASE-ROOT \ - {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} \ - {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} \ - {% endif %}{% else %} \ - {% endif %}--report-output $@ -{%- endif %} - -check_for_robot_updates: -{%- if project.robot.report.custom_profile %} - @odk-helper check-robot-profile -{%- else %} - echo "You are not using a custom profile, so you are getting the joy of the latest ROBOT report!" -{% endif %} # ---------------------------------------- # Release assets From fff94faaee602d243a15839aecaec78b35b41508 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Fri, 14 Aug 2026 22:16:48 +0100 Subject: [PATCH 05/25] Regroup and re-format the entire imports section. The code related to imports was split over three sections, found at different places in the Makefile: one section for the import modules themselves (the code that produces the actual modules), one section for the generation of the seed, and one section for mirroring the remote ontologies. We regroup all this code in one large "imports" section, containing one subsection for mirrors, one subsection for the seed (which may in fact be skipped entirely if there is no seed to generate), and one subsection for the imports. As part of those moves: * the code that produces the "simple seed" is moved to the section about the release artefacts, since this is only place where that seed (which has nothing to about imports) is needed; * the code that produces the SRCMERGED and EDIT_PREPROCESSED intermediate files is moved to a dedicated section, since on the contrary those files are used for many more things than just preparing the import modules. --- .../templates/src/ontology/Makefile.jinja2 | 776 +++++++++--------- 1 file changed, 406 insertions(+), 370 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 71757d8..a0ac01b 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -72,7 +72,6 @@ CATALOG = {{ project.catalog_file }} CONTEXT_FILE = config/context.json {% endif -%} EXTENDED_PREFIX_MAP = $(TMPDIR)/obo.epm.json -ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt EDIT_PREPROCESSED = $(TMPDIR)/$(ONT)-preprocess.owl SRCMERGED = $(TMPDIR)/merged-$(ONT)-edit.ofn @@ -235,6 +234,25 @@ $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar: {% endfor -%} {% endif %} +# ---------------------------------------- +# Internal intermediate products +# ---------------------------------------- + +{# This section is intended to group the rules that build intermediate + files that are used in several sections throughout the Makefile. -#} + +# The -edit file plus all components, but without any import module. +$(SRCMERGED): $(EDIT_PREPROCESSED) $(OTHER_SRC) + $(ROBOT) remove --input $< --select imports --trim false \ + merge $(foreach src, $(OTHER_SRC), --input $(src)) \ + convert --format ofn --output $@ + +# Override this rule in {{ project.id }}.Makefile to perform any needed +# custom preprocessing on the -edit file. +$(EDIT_PREPROCESSED): $(SRC) + $(ROBOT) convert --input $< --format ofn --output $@ + + # ---------------------------------------- # Quality control # ---------------------------------------- @@ -330,6 +348,376 @@ ifneq ($(SPARQL_VALIDATION_QUERIES),) endif +{% if project.import_group is defined -%} +# ---------------------------------------- +# Imports +# ---------------------------------------- + +IMPORTS ={% for imp in project.import_group.products %} {{ imp.id }}{% endfor %} + +# Mirrors +# ---------------------------------------- + +ifeq ($(MIR),true) +{% for ont in project.import_group.products -%} + +## ONTOLOGY: {{ ont.id }} +{% if ont.description -%} +## {{ ont.description }} +{% endif -%} + +{% if ont.mirror_type != "no_mirror" -%} +.PHONY: download-mirror-{{ ont.id }} +download-mirror-{{ ont.id }}: | $(TMPDIR) + odk-helper download --output $(TMPDIR)/$@.owl \ + --reference $(MIRRORDIR)/{{ ont.id }}.owl \ + --max-retry {{ project.import_group.mirror_retry_download }} \ + {% if ont.mirror_from -%} + {{ ont.mirror_from }} + {%- elif ont.use_variant is not none -%} + $(OBOBASE)/{{ ont.id }}/{{ ont.id }}-{{ ont.use_variant }}.owl{% if ont.use_gzipped %}.gz{% endif %} + {%- else -%} + $(OBOBASE)/{{ ont.id }}.owl{% if ont.use_gzipped %}.gz{% endif %} + {%- endif %} + +{% if ont.mirror_type == 'custom' -%} +$(MIRRORDIR)/{{ ont.id }}.owl: + @echo "ERROR: You have configured your default mirror type to be custom; + @echo " this behavior needs to be overwritten in {{ project.id }}.Makefile!" + @false +{% else -%} +{% if ont.is_large -%} +ifeq ($(IMP_LARGE),true) +{% endif -%} +$(MIRRORDIR)/{{ ont.id }}.owl: download-mirror-{{ ont.id }} + @if [ -f $(TMPDIR)/download-mirror-{{ ont.id }}.owl ]; then \ + $(ROBOT) + {%- if ont.make_base or "base" == ont.mirror_type -%} + {# #} remove --input $(TMPDIR)/download-mirror-{{ ont.id }}.owl + {%- if ont.base_iris is not none %} \ + {%- for iri in ont.base_iris %} + --base-iri {{ iri }} \ + {%- endfor -%} + {%- else %} \ + --base-iri $(OBOBASE)/{{ ont.id.upper() }} \ + {%- endif %} + --axioms external --preserve-structure false --trim false \ + convert --format ofn --output $@ ; \ + {%- else -%} + {# #} convert --input $(TMPDIR)/download-mirror-{{ ont.id }}.owl \ + --format ofn --output $@ ; \ + {%- endif %} + fi +{% if ont.is_large -%} +endif +{% endif -%} + +{% endif -%} +.PRECIOUS: $(MIRRORDIR)/{{ ont.id }}.owl + +{% endif %}{# !ont.mirror_type != 'no_mirror' -#} + +{% endfor %}{# for ont in project.import_group.products -#} + +{% if project.import_group.use_base_merging -%} +ALL_MIRRORS = $(foreach imp, $(IMPORTS), $(MIRRORDIR)/$(imp).owl) +MERGE_MIRRORS = true + +ifeq ($(MERGE_MIRRORS),true) +$(MIRRORDIR)/merged.owl: $(ALL_MIRRORS) + $(ROBOT) merge $(foreach mirror, $^, --input $(mirror)) \ + {%- if project.import_group.annotate_defined_by %} + --annotate-defined-by true \ + {%- endif %} + {%- if project.import_group.base_merge_drop_equivalent_class_axioms %} + remove --axioms equivalent --preserve-structure false \ + {%- endif %} + convert --format ofn --output $@ + +.PRECIOUS: $(MIRRORDIR)/merged.owl +endif +{% endif -%} + +endif # MIR=true + +{% if project.import_group.scan_signature or project.use_custom_import_module -%} +# Imports seeding system +# ---------------------------------------- + +{% if project.import_group.scan_signature -%} +# All entities referenced from the -edit file and any component +PRESEED = $(TMPDIR)/pre_seed.txt +{% endif -%} +{% if project.use_custom_import_module -%} +# Template-defined custom seed +IMPORT_MODULE_SIGNATURE = $(TMPDIR)/external_import_terms.txt +IMPORT_MODULE_TEMPLATE = $(TEMPLATEDIR)/external_import.tsv +{# FIXME: What is the purpose of that by-product? + see https://github.com/INCATools/ontology-development-kit/issues/1365 -#} +IMPORT_MODULE = $(IMPORTDIR)/external_import.owl +{% endif -%} +# All entities to import in addition to defined in the individual _terms.txt files +IMPORTSEED = $(TMPDIR)/seed.txt +T_IMPORTSEED = --term-file $(IMPORTSEED) + +{% if project.import_group.scan_signature -%} +$(PRESEED): $(SRCMERGED) | $(TMPDIR) + $(ROBOT) query --input $< --format --csv \ + --query $(SPARQLDIR)/terms.sparql $@ + +{% endif -%} + +{% if project.use_custom_import_module -%} +$(IMPORT_MODULE_SIGNATURE) $(IMPORT_MODULE): $(IMPORT_MODULE_TEMPLATE) | $(TMPDIR) + $(ROBOT) template --template $< \ + --ontology-iri "$(ONTBASE)/external_import.owl" \ + convert --format {{ project.import_component_format|default("ofn") }} \ + --output $(IMPORT_MODULE) \ + query --format csv \ + --query $(SPARQLDIR)/terms.sparql $(IMPORT_MODULE_SIGNATURE) + +{% endif -%} + +$(IMPORTSEED): +{%- if project.import_group.scan_signature %} $(PRESEED) +{%- if project.use_dosdps %} $(TMPDIR)/all_pattern_terms.txt{% endif -%} +{%- endif -%} +{%- if project.use_custom_import_module %} $(IMPORT_MODULE_SIGNATURE){% endif %} | $(TMPDIR) + cat $^ | sort | uniq > $@ + +{% endif %}{# project.import_group.scan_signature or project.use_custom_import_module -#} + +# Import modules +# ---------------------------------------- + +ifeq ($(IMP),true) + +{% if project.import_group.strip_annotation_properties -%} +ANNOTATION_PROPERTIES ={% for p in project.import_group.annotation_properties %} {{ p }}{% endfor %} +{% endif -%} +{% if project.import_group.use_base_merging -%} +ALL_TERMS = $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_terms.txt) + +{% if 'slme' == project.import_group.module_type -%} +$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ + $(IMPORTSEED) | all_robot_plugins + $(ROBOT) merge --input $<{% if project.import_group.exclude_iri_patterns is not none -%} + {%- for pattern in project.import_group.exclude_iri_patterns %} \ + remove --select "{{ pattern }}"{% endfor %}{% endif %} \ + extract $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ + --force true --copy-ontology-annotations false \ + --individuals {{ project.import_group.slme_individuals }} \ + --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ + --select complement --select annotation-properties \{% endif %} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) + +{% else -%} +$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ + $(IMPORTSEED) + @echo "ERROR: You have configured the merged import as a custom module;" + @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" + @false + +{% endif -%} +{% else %}{# !project.import_group.use_base_merging -#} + +## Default module type ({{ project.import_group.module_type }}) +{% if 'slme' == project.import_group.module_type -%} +$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ + $(IMPORTSEED) | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --force true --copy-ontology-annotations true \ + --individuals {{ project.import_group.slme_individuals }} \ + --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --select complement --select annotation-properties \{% endif %} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) + +{% elif 'minimal' == project.import_group.module_type -%} +$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ + $(IMPORTSEED) | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --force true --copy-ontology-annotations true \ + --method BOT \ + remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ + --axioms external \ + --preserve-structure false --trim false \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --select complement \ + --select "classes individuals annotation-properties" \ + $(ANNOTATE_CONVERT_FILE) + +{% elif 'mirror' == project.import_group.module_type -%} +$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + --add-source true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) + +{% elif 'filter' == project.import_group.module_type -%} +$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ + $(IMPORTSEED) | all_robot_plugins + $(ROBOT) merge --input $< \ + annotate --remove-annotations \ + odk:normalize --add-source true \ + remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ + --axioms external \ + --preserve-structure false --trim false \ + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --select complement \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) + +{% elif 'custom' == project.import_group.module_type -%} +$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl + @echo "ERROR: You have configured the default module type to be custom;" + @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" + @false + +{% endif -%} +.PRECIOUS: $(IMPORTDIR)/%_import.owl + +{# + # An import module of a different type than the default type needs its + # own specific rule to override the pattern rule above. + # A module that is marked as "large" (regardless of its type) also + # needs its own rule, so that it can be selectively excluded when + # running under IMP_LARGE=false. + # We generate those rules here. +-#} +{% for ont in project.import_group.special_products -%} +## Module for ontology: {{ ont.id }} ({{ ont.module_type }}) +{% if ont.is_large -%} +ifeq ($(IMP_LARGE),true) +{% endif -%} +{% if 'slme' == ont.module_type -%} +$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --copy-ontology-annotations true --force true \ + --individuals {{ ont.slme_individuals }} \ + --method {{ ont.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} + --term {{ p }} \{% endfor %} + --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --select complement --select annotation-properties \{% endif %} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) +{% elif 'filter' == ont.module_type -%} +$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --copy-ontology-annotations true --force true --method BOT \ + remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} + --base-iri {{ iri }} \{% endfor %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} + --term {{ p }} \{% endfor %} + --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --select complement \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) +{% elif 'mirror' == ont.module_type -%} +$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + --add-source true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) +{% elif 'minimal' == ont.module_type -%} +$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins + $(ROBOT) annotate --input $< --remove-annotations \ + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --copy-ontology-annotations true --force true --method BOT \ + remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} + --base-iri {{ iri }} \{% endfor %} + odk:normalize --base-iri {{ project.uribase }} --subset-decls true \ + --synonym-decls true \ + repair --merge-axiom-annotations true \ + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} + --term {{ p }} \{% endfor %} + --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --select complement \ + --select "classes individuals annotation-properties" \ + $(ANNOTATE_CONVERT_FILE) +{% elif 'custom' -%} +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if 'no_mirror' != ont.mirror_type %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} + @echo "ERROR: You have configured {{ ont.id }} as a custom module;" + @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" + @false +{% endif -%} +{% if ont.is_large -%} +endif # IMP_LARGE=true +{% endif %} +{% endfor -%} +{% endif %}{# !use_base_merging -#} + +{% if project.import_group.export_obo -%} +# Convert any import module to OBO (this can be useful for spot-checks +# and diffs). +$(IMPORTDIR)/%_import.obo: $(IMPORTDIR)/%_import.owl + $(ROBOT) convert --input $< --check false --format obo --output $@ + +{% endif -%} +endif # IMP=true + +.PHONY: refresh-imports +refresh-imports: + $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=true clean all_imports + +.PHONY: no-mirror-refresh-imports +no-mirror-refresh-imports: + $(MAKE) --assume-new=$(SRC) \ + $(foreach imp, $(IMPORTS), --assume-new=$(IMPORTDIR)/$(imp)_terms.txt) \ + IMP=true MIR=false PAT=false IMP_LARGE=true all_imports + +.PHONY: refresh-imports-excluding-large +refresh-imports-excluding-large: + $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=false clean all_imports + +.PHONY: refresh-% +refresh-%: + rm -f $(TMPDIR)/download-mirror-$*.owl.info + $(MAKE) --assume-new=$(SRC) --assume-new=$(IMPORTDIR)/$*_terms.txt \ + IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl + +.PHONY: no-mirror-refresh-% +no-mirror-refresh-%: + $(MAKE) --assume-new=$(SRC) --assume-new=$(IMPORTDIR)/$*_terms.txt \ + IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl + + +{% endif %}{# !project.import_group is defined -#} + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -345,11 +733,6 @@ all_main: $(MAIN_FILES) # Import assets # ---------------------------------------- -{% if project.import_group is defined %} -IMPORTS = {% for imp in project.import_group.products %} {{ imp.id }}{% endfor %} -{% else %} -IMPORTS = -{% endif %} IMPORT_ROOTS = {% if project.import_group.use_base_merging %} $(IMPORTDIR)/merged_import{% else %}$(patsubst %, $(IMPORTDIR)/%_import, $(IMPORTS)){% endif %}{% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import{% endif %} IMPORT_OWL_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).owl) {%- if project.import_group.export_obo %} @@ -502,303 +885,6 @@ $(TMPDIR)/current-release.owl: $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ -# ------------------------ -# Imports: Seeding system -# ------------------------ - -{% if project.import_group.scan_signature -%} -# pre_seed.txt contains all entities referenced from the -edit file -# and its components -PRESEED=$(TMPDIR)/pre_seed.txt -$(PRESEED): $(SRCMERGED) - $(ROBOT) query --input $< --format --csv \ - --query $(SPARQLDIR)/terms.sparql $@ - -{% endif -%} -$(SRCMERGED): $(EDIT_PREPROCESSED) $(OTHER_SRC) - $(ROBOT) remove --input $< --select imports --trim false \ - merge $(foreach src, $(OTHER_SRC), --input $(src)) \ - convert --format ofn --output $@ - -$(EDIT_PREPROCESSED): $(SRC) - $(ROBOT) convert --input $< --format ofn --output $@ - -{% if 'basic' in project.release_artefacts or 'simple' in project.release_artefacts or 'simple-non-classified' in project.release_artefacts or project.primary_release == 'basic' or project.primary_release == 'simple-non-classified' or project.primary_release == 'simple' -%} -SIMPLESEED=$(TMPDIR)/simple_seed.txt - -$(SIMPLESEED): $(SRCMERGED) $(ONTOLOGYTERMS) - $(ROBOT) query -f csv -i $< --query ../sparql/simple-seed.sparql $@.tmp &&\ - cat $@.tmp $(ONTOLOGYTERMS) | sort | uniq > $@ &&\ - echo "http://www.geneontology.org/formats/oboInOwl#SubsetProperty" >> $@ &&\ - echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ - -{% endif -%} -{% if project.use_custom_import_module -%} -IMPORT_MODULE_TEMPLATE=$(TEMPLATEDIR)/external_import.tsv -IMPORT_MODULE_SIGNATURE=$(TMPDIR)/external_import_terms.txt -IMPORT_MODULE=$(IMPORTDIR)/external_import.owl -$(IMPORT_MODULE): $(IMPORT_MODULE_TEMPLATE) | $(TMPDIR) - $(ROBOT) template --template $< {% if project.use_context %}--add-prefixes $(CONTEXT_FILE) {% endif %}\ - --ontology-iri "$(ONTBASE)/external_import.owl" \ - convert -f {{ project.import_component_format|default('ofn') }} \ - --output $@ - -$(IMPORT_MODULE_SIGNATURE): $(IMPORT_MODULE) | $(TMPDIR) - $(ROBOT) query -f csv -i $< --query ../sparql/terms.sparql $@.tmp &&\ - cat $@.tmp | sort | uniq > $@ - -{% endif -%} - -{% if project.import_group.scan_signature or project.use_custom_import_module -%} -# seed.txt contains all entities to import in addition to those defined -# in the individual _terms.txt files. -IMPORTSEED = $(TMPDIR)/seed.txt -$(IMPORTSEED): -{%- if project.import_group.scan_signature %} $(PRESEED) -{%- if project.use_dosdps %} $(TMPDIR)/all_pattern_terms.txt{% endif -%} -{%- endif -%} -{%-if project.use_custom_import_module %} $(IMPORT_MODULE_SIGNATURE){% endif %} | $(TMPDIR) - cat $^ | sort | uniq > $@ - -T_IMPORTSEED = --term-file $(IMPORTSEED) - -{% endif -%} - -{% if project.import_group is defined -%} -{% if project.import_group.strip_annotation_properties -%} -ANNOTATION_PROPERTIES={% for p in project.import_group.annotation_properties %}{{ p }} {% endfor %} - -{% endif -%} -# ---------------------------------------- -# Import modules -# ---------------------------------------- -# Most ontologies are modularly constructed using portions of other ontologies -# These live in the imports/ folder -# This pattern uses ROBOT to generate an import module - -ifeq ($(IMP),true) - -{% if project.import_group.use_base_merging -%} -ALL_TERMS = $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_terms.txt) - -{% if 'slme' == project.import_group.module_type -%} -$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ - $(IMPORTSEED) | all_robot_plugins - $(ROBOT) merge --input $<{% if project.import_group.exclude_iri_patterns is not none -%} - {%- for pattern in project.import_group.exclude_iri_patterns %} \ - remove --select "{{ pattern }}"{% endfor %}{% endif %} \ - extract $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ - --force true --copy-ontology-annotations false \ - --individuals {{ project.import_group.slme_individuals }} \ - --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) - -{% else -%} -$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ - $(IMPORTSEED) - @echo "ERROR: You have configured the merged import as a custom module;" - @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" - @false - -{% endif -%} -{% else %}{# !project.import_group.use_base_merging -#} - -## Default module type ({{ project.import_group.module_type }}) -{% if 'slme' == project.import_group.module_type -%} -$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ - $(IMPORTSEED) | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --force true --copy-ontology-annotations true \ - --individuals {{ project.import_group.slme_individuals }} \ - --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'minimal' == project.import_group.module_type -%} -$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ - $(IMPORTSEED) | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --force true --copy-ontology-annotations true \ - --method BOT \ - remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ - --axioms external \ - --preserve-structure false --trim false \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement \ - --select "classes individuals annotation-properties" \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'mirror' == project.import_group.module_type -%} -$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - --add-source true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'filter' == project.import_group.module_type -%} -$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ - $(IMPORTSEED) | all_robot_plugins - $(ROBOT) merge --input $< \ - annotate --remove-annotations \ - odk:normalize --add-source true \ - remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ - --axioms external \ - --preserve-structure false --trim false \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'custom' == project.import_group.module_type -%} -$(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl - @echo "ERROR: You have configured the default module type to be custom;" - @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" - @false - -{% endif -%} -.PRECIOUS: $(IMPORTDIR)/%_import.owl - -{# - # An import module of a different type than the default type needs its - # own specific rule to override the pattern rule above. - # A module that is marked as "large" (regardless of its type) also - # needs its own rule, so that it can be selectively excluded when - # running under IMP_LARGE=false. - # We generate those rules here. --#} -{% for ont in project.import_group.special_products -%} -## Module for ontology: {{ ont.id }} ({{ ont.module_type }}) -{% if ont.is_large -%} -ifeq ($(IMP_LARGE),true) -{% endif -%} -{% if 'slme' == ont.module_type -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --copy-ontology-annotations true --force true \ - --individuals {{ ont.slme_individuals }} \ - --method {{ ont.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} - --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'filter' == ont.module_type -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --copy-ontology-annotations true --force true --method BOT \ - remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} - --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'mirror' == ont.module_type -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - --add-source true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'minimal' == ont.module_type -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins - $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --copy-ontology-annotations true --force true --method BOT \ - remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %} - odk:normalize --base-iri {{ project.uribase }} --subset-decls true \ - --synonym-decls true \ - repair --merge-axiom-annotations true \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} - --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement \ - --select "classes individuals annotation-properties" \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'custom' -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if 'no_mirror' != ont.mirror_type %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} - @echo "ERROR: You have configured {{ ont.id }} as a custom module;" - @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" - @false -{% endif -%} -{% if ont.is_large -%} -endif # IMP_LARGE=true -{% endif %} -{% endfor -%} -{% endif %}{# !use_base_merging -#} - -{% if project.import_group.export_obo -%} -# Convert any import module to OBO (this can be useful for spot-checks -# and diffs). -$(IMPORTDIR)/%_import.obo: $(IMPORTDIR)/%_import.owl - $(ROBOT) convert --input $< --check false --format obo --output $@ - -{% endif -%} -endif # IMP=true - -.PHONY: refresh-imports -refresh-imports: - $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=true clean all_imports - -.PHONY: no-mirror-refresh-imports -no-mirror-refresh-imports: - $(MAKE) --assume-new=$(SRC) \ - $(foreach imp,$(IMPORTS),--assume-new=$(IMPORTDIR)/$(imp)_terms.txt) \ - IMP=true MIR=false PAT=false IMP_LARGE=true all_imports - -.PHONY: refresh-imports-excluding-large -refresh-imports-excluding-large: - $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=false clean all_imports - -.PHONY: refresh-% -refresh-%: - rm -f $(TMPDIR)/download-mirror-$*.owl.info - $(MAKE) --assume-new=$(SRC) --assume-new=$(IMPORTDIR)/$*_terms.txt \ - IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl - -.PHONY: no-mirror-refresh-% -no-mirror-refresh-%: - $(MAKE) --assume-new=$(SRC) --assume-new=$(IMPORTDIR)/$*_terms.txt \ - IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl - -{% endif %}{# !project.import_group is defined -#} {% if project.orcidio_support -%} # ---------------------------------------- @@ -917,71 +1003,6 @@ $(COMPONENTSDIR)/{{ component.filename }}:{% for mapping in component.mappings % endif # COMP=true {% endif %}{# ! project.components is not none -#} -{% if project.import_group is defined -%} -# ---------------------------------------- -# Mirroring upstream ontologies -# ---------------------------------------- - -ifeq ($(MIR),true) -{% for ont in project.import_group.products -%} - -## ONTOLOGY: {{ ont.id }} -{% if ont.description -%} -## {{ ont.description }} -{% endif -%} - -{% if ont.mirror_type != 'no_mirror' -%} -.PHONY: download-mirror-{{ ont.id }} -.PRECIOUS: $(MIRRORDIR)/{{ ont.id }}.owl -download-mirror-{{ ont.id }}: | $(TMPDIR) - odk-helper download --output $(TMPDIR)/$@.owl \ - --reference $(MIRRORDIR)/{{ ont.id }}.owl \ - --max-retry {{ project.import_group.mirror_retry_download }} \ - {% if ont.mirror_from %}{{ ont.mirror_from }}{% elif ont.use_variant is not none %}$(OBOBASE)/{{ ont.id }}/{{ ont.id }}-{{ ont.use_variant }}.owl{% if ont.use_gzipped %}.gz{% endif %}{% else %}$(OBOBASE)/{{ ont.id }}.owl{% if ont.use_gzipped %}.gz{% endif %}{% endif %} - -{% if ont.mirror_type == 'custom' -%} -$(MIRRORDIR)/{{ ont.id }}.owl: - @echo "ERROR: You have configured your default mirror type to be custom; - @echo " this behavior needs to be overwritten in {{ project.id }}.Makefile!" - @false - -{% else -%} -{% if ont.is_large -%} -ifeq ($(IMP_LARGE),true) -{% endif -%} -$(MIRRORDIR)/{{ ont.id }}.owl: download-mirror-{{ ont.id }} - @if [ -f $(TMPDIR)/download-mirror-{{ ont.id }}.owl ]; then \ - $(ROBOT) {% if ont.make_base or 'base' == ont.mirror_type %}remove -i $(TMPDIR)/download-mirror-{{ ont.id }}.owl \{% if ont.base_iris is not none %}{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %}{% else %} - --base-iri $(OBOBASE)/{{ ont.id.upper() }} \{% endif %} - --axioms external --preserve-structure false --trim false \ - convert {% else %}convert -i $(TMPDIR)/download-mirror-{{ ont.id }}.owl {% endif %}--format ofn -o $@ ; \ - fi - -{% if ont.is_large -%} -endif -{% endif -%} - -{% endif -%} - -{% endif %}{# !ont.mirror_type != 'no_mirror' -#} - -{% endfor %}{# for ont in project.import_group.products -#} - -{% if project.import_group.use_base_merging -%} -ALL_MIRRORS = $(patsubst %, $(MIRRORDIR)/%.owl, $(IMPORTS)) -MERGE_MIRRORS = true - -ifeq ($(MERGE_MIRRORS),true) -$(MIRRORDIR)/merged.owl: $(ALL_MIRRORS) - $(ROBOT) merge $(foreach mirror,$^,-i $(mirror)) {% if project.import_group.annotate_defined_by %}--annotate-defined-by true{% endif %} {% if project.import_group.base_merge_drop_equivalent_class_axioms %}remove --axioms equivalent --preserve-structure false {% endif %}\ - convert --format ofn -o $@ -.PRECIOUS: $(MIRRORDIR)/merged.owl -endif -{% endif -%} - -endif # MIR=true -{% endif %}{# project.import_group is defined -#} {% if project.subset_group is defined %} # ---------------------------------------- @@ -1432,8 +1453,23 @@ $(ONT).json: $(ONT).owl # ----------------------------------------------------- SHARED_ROBOT_COMMANDS = {% if project.remove_owl_nothing -%}remove --term owl:Nothing{% endif %} -$(ONTOLOGYTERMS): $(SRCMERGED) - $(ROBOT) query -f csv -i $< --query ../sparql/{{ project.id }}_terms.sparql $@ +{% if "basic" in project.release_artefacts + or "simple" in project.release_artefacts + or "simple-non-classified" in project.release_artefacts + or project.primary_release == "basic" + or project.primary_release == "simple" + or project.primary_release == "simple-non-classified" -%} +SIMPLESEED = $(TMPDIR)/simple_seed.txt + +$(SIMPLESEED): $(SRCMERGED) + $(ROBOT) query --input $< --format csv \ + --query $(SPARQLDIR)/simple-seed.sparql $@.1.tmp \ + --query $(SPARQLDIR)/{{ project.id }}_terms.sparql $@.2.tmp + cat $@.1.tmp $@.2.tmp | sort | uniq > $@ + echo "http://www.geneontology.org/formats/oboInOwl#SubsetProperty" >> $@ + echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ + +{% endif -%} {% for format in project.export_formats -%} {% if project.gzip_main -%} From 74b38db000908974268a38ee7c2758e6cd24e457 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sat, 15 Aug 2026 13:03:26 +0100 Subject: [PATCH 06/25] Move components section. Components are pretty similar to import modules, so we move their section just after the imports section. --- .../templates/src/ontology/Makefile.jinja2 | 166 +++++++++--------- 1 file changed, 87 insertions(+), 79 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index a0ac01b..301f1d5 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -718,6 +718,93 @@ no-mirror-refresh-%: {% endif %}{# !project.import_group is defined -#} +{% if project.components is not none -%} +# ---------------------------------------- +# Components +# ---------------------------------------- +# Some ontologies contain external and internal components. +# A component is included in the ontology in its entirety. + +ifeq ($(COMP),true) + +$(COMPONENTSDIR)/%.owl: $(TMPDIR)/stamp-component-%.owl | $(COMPONENTSDIR) + test -f $@ || touch $@ +.PRECIOUS: $(COMPONENTSDIR)/%.owl + +$(TMPDIR)/stamp-component-%.owl: | $(TMPDIR) + touch $@ +.PRECIOUS: $(TMPDIR)/stamp-component-%.owl + +{% for component in project.components.products -%} +{% if component.source is not none -%} +ifeq ($(MIR),true) +.PHONY: download-component-{{ component.filename }} +download-component-{{ component.filename }}: | $(TMPDIR) + @odk-helper download --output $(TMPDIR)/$@ \ + --reference $(COMPONENTSDIR)/{{ component.filename }} \ + {{ component.source }} + +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }} download-component-{{ component.filename }} + @if [ -f $(TMPDIR)/download-component-{{ component.filename }} ]; then \ + $(ROBOT) merge -i $(TMPDIR)/download-component-{{ component.filename }} \{% if component.make_base %} + remove {% for iri in component.base_iris %}--base-iri {{ iri }} \ + {% endfor %}--axioms external --preserve-structure false --trim false \{% endif %} + $(ANNOTATE_CONVERT_FILE) ; \ + fi + +.PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} +endif # MIR=true + +{% elif component.use_template -%} +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }}{% for template in component.templates %} \ + $(TEMPLATEDIR)/{{ template }}{% endfor %} + $(ROBOT) template {% if component.template_options is not none %}{{ component.template_options }}{% endif %} \ + {% for template in component.templates %}--template $(TEMPLATEDIR)/{{ template }} \ + {% endfor %}$(ANNOTATE_CONVERT_FILE) +.PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} + +{% elif component.use_mappings -%} +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }}{% for mapping in component.mappings %} \ + $(MAPPINGDIR)/{{ mapping }}{% endfor %} \ + | all_robot_plugins + $(ROBOT) --add-prefix 'sssom: https://w3id.org/sssom/' \ + --add-prefix 'semapv: http://w3id.org/semapv/vocab/' \ + sssom:inject {% for mapping in component.mappings %}--sssom $(MAPPINGDIR)/{{ mapping }} \ + {% endfor %}--create --direct \ + $(ANNOTATE_CONVERT_FILE) +.PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} + +{% endif -%} +{% endfor -%} +endif # COMP=true + +.PHONY: all_components +all_components: $(OTHER_SRC) + +.PHONY: recreate-components +recreate-components: + $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ + {% endfor %}COMP=true IMP=false MIR=true PAT=true all_components + +.PHONY: no-mirror-recreate-components +no-mirror-recreate-components: + $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ + {% endfor %}COMP=true IMP=false MIR=false PAT=true all_components + +.PHONY: recreate-% +recreate-%: + rm -f $(TMPDIR)/download-component-$*.owl.info + $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ + COMP=true IMP=false MIR=true PAT=true $(COMPONENTSDIR)/$*.owl + +.PHONY: no-mirror-recreate-% +no-mirror-recreate-%: + $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ + COMP=true IMP=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl + + +{% endif %}{# ! project.components is not none -#} + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -923,85 +1010,6 @@ endif {% endif -%} -{% if project.components is not none -%} -# ---------------------------------------- -# Components -# ---------------------------------------- -# Some ontologies contain external and internal components. A component is included in the ontology in its entirety. - -ifeq ($(COMP),true) -.PHONY: all_components -all_components: $(OTHER_SRC) - -.PHONY: recreate-components -recreate-components: - $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ - {% endfor %}COMP=true IMP=false MIR=true PAT=true all_components - -.PHONY: no-mirror-recreate-components -no-mirror-recreate-components: - $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ - {% endfor %}COMP=true IMP=false MIR=false PAT=true all_components - -.PHONY: recreate-% -recreate-%: - rm -f $(TMPDIR)/download-component-$*.owl.info - $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ - COMP=true IMP=false MIR=true PAT=true $(COMPONENTSDIR)/$*.owl - -.PHONY: no-mirror-recreate-% -no-mirror-recreate-%: - $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ - COMP=true IMP=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl - -$(COMPONENTSDIR)/%.owl: $(TMPDIR)/stamp-component-%.owl | $(COMPONENTSDIR) - test -f $@ || touch $@ -.PRECIOUS: $(COMPONENTSDIR)/%.owl - -$(TMPDIR)/stamp-component-%.owl: | $(TMPDIR) - touch $@ -.PRECIOUS: $(TMPDIR)/stamp-component-%.owl - -{% for component in project.components.products -%} -{% if component.source is not none -%} -ifeq ($(MIR),true) -.PHONY: download-component-{{ component.filename }} -download-component-{{ component.filename }}: | $(TMPDIR) - @odk-helper download --output $(TMPDIR)/$@ \ - --reference $(COMPONENTSDIR)/{{ component.filename }} \ - {{ component.source }} - -$(COMPONENTSDIR)/{{ component.filename }}: download-component-{{ component.filename }} $(TMPDIR)/stamp-component-{{ component.filename }} - @if [ -f $(TMPDIR)/download-component-{{ component.filename }} ]; then \ - $(ROBOT) merge -i $(TMPDIR)/download-component-{{ component.filename }} \{% if component.make_base %} - remove {% for iri in component.base_iris %}--base-iri {{ iri }} \ - {% endfor %}--axioms external --preserve-structure false --trim false \{% endif %} - $(ANNOTATE_CONVERT_FILE) ; \ - fi - -.PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} -endif # MIR=true - -{% elif component.use_template -%} -$(COMPONENTSDIR)/{{ component.filename }}:{% for template in component.templates %} $(TEMPLATEDIR)/{{ template }}{% endfor %} $(TMPDIR)/stamp-component-{{ component.filename }} - $(ROBOT) template {% if component.template_options is not none %}{{ component.template_options }}{% endif %} \ - {% for template in component.templates %}--template $(TEMPLATEDIR)/{{ template }} \ - {% endfor %}$(ANNOTATE_CONVERT_FILE) -.PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} - -{% elif component.use_mappings -%} -$(COMPONENTSDIR)/{{ component.filename }}:{% for mapping in component.mappings %} $(MAPPINGDIR)/{{ mapping }}{% endfor %} $(TMPDIR)/stamp-component-{{ component.filename }} | all_robot_plugins - $(ROBOT) --add-prefix 'sssom: https://w3id.org/sssom/' \ - --add-prefix 'semapv: http://w3id.org/semapv/vocab/' \ - sssom:inject {% for mapping in component.mappings %}--sssom $(MAPPINGDIR)/{{ mapping }} \ - {% endfor %}--create --direct \ - $(ANNOTATE_CONVERT_FILE) -.PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} - -{% endif -%} -{% endfor -%} -endif # COMP=true -{% endif %}{# ! project.components is not none -#} {% if project.subset_group is defined %} From 8bd9a0e8c2667feeaa6f0a44d0f90db79fce4f9d Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sat, 15 Aug 2026 14:59:35 +0100 Subject: [PATCH 07/25] Move and re-format the mappings and bridges sections. Move the sections about SSSOM mappings and bridges into a single "mappings and bridges" section. As part of the "re-formatting", also apply to the downloading of SSSOM sets the same logic as the one used for the downloading of mirrors and components -- ensuring that (i) a remote mapping set is automatically refreshed under MIR=true (previously, once downloaded a first time, a mapping set was never automatically refreshed), and (ii) if the refreshed mapping set has not in fact changed since the last download, the refresh does not trigger the rebuilding of any file that depends on the mapping set. --- .../templates/src/ontology/Makefile.jinja2 | 206 ++++++++++-------- 1 file changed, 112 insertions(+), 94 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 301f1d5..d196ec0 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -805,6 +805,118 @@ no-mirror-recreate-%: {% endif %}{# ! project.components is not none -#} +{% if project.use_mappings or project.bridge_group is not none -%} +# ---------------------------------------- +# Mappings and bridges +# ---------------------------------------- + +{% if project.use_mappings -%} +# SSSOM mappings sets +# ---------------------------------------- + +MAPPING_TESTER = sssom validate +SSSOMPY = sssom + +validate-sssom-%: + tsvalid $(MAPPINGDIR)/$*.sssom.tsv --comment "#" + $(MAPPING_TESTER) $(MAPPINGDIR)/$*.sssom.tsv + +validate_mappings: + $(MAKE_FAST) $(foreach n, $(MAPPINGS), validate-sssom-$(n)) + +normalize-sssom-%: + sssom-cli --output $(MAPPINGDIR)/$*.sssom.tsv $(MAPPINGDIR)/$*.sssom.tsv + +normalize_mappings: + $(MAKE_FAST) $(foreach n, $(MAPPINGS), normalize-sssom-$(n)) + +{% if project.sssom_mappingset_group is not none -%} +{% for mapping in project.sssom_mappingset_group.products -%} +{% if mapping.maintenance == "extract" -%} +{% if project.sssom_mappingset_group.mapping_extractor == "sssom-py" -%} +$(TMPDIR)/{{ mapping.id }}.obographs.json: {{ mapping.source_file }} + $(ROBOT) annotate --input $< \ + --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format json --output $@ + +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: $(TMPDIR)/{{ mapping.id }}.obographs.json + $(SSSOMPY) parse $< --input-format obographs-json {% if mapping.sssom_tool_options %}\ + {{ mapping.sssom_tool_options }} \ + {% else %}{% endif %}--output $@ + +{% elif project.sssom_mappingset_group.mapping_extractor == "sssom-java" -%} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: {{ mapping.source_file }} | all_robot_plugins + $(ROBOT) sssom:xref-extract --input $< --replace \ + --ignore-treat-xrefs --all-xrefs \ + --mapping-file $@ + +{% endif -%} +{% elif mapping.maintenance == "manual" -%} +# This mappingset is manually curated, so we only check that the file actually exists. +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: + test -f $@ + +{% elif mapping.maintenance == "merged" -%} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% if mapping.source_mappings|length >=1 %} $(MAPPINGDIR)/{{ mapping.source_mappings[0] }}.sssom.tsv{% endif %} + {%- for source in mapping.source_mappings[1:] %} \ + $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} + sssom-cli --output $@ $^ + +{% elif mapping.maintenance == "mirror" -%} +ifeq ($(MIR),true) +.PHONY: download-sssom-{{ mapping.id }} +download-sssom-{{ mapping.id }}: | $(TMPDIR) + odk-helper download --output $(TMPDIR)/download-sssom-{{ mapping.id }}.sssom.tsv \ + --reference $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv \ + {{ mapping.mirror_from }} + +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: download-sssom-{{ mapping.id }} + @if [ -f $(TMPDIR)/download-sssom-{{ mapping.id }}.sssom.tsv ]; then \ + cp $(TMPDIR)/download-sssom-{{ mapping.id }}.sssom.tsv $@ ; \ + fi +endif + +{% elif mapping.maintenance == "custom" -%} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: + @echo "ERROR: You have configured {{ mapping.id }} as a custom mapping set;" + @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" + @false + +{% endif -%} +{% endfor -%} +{% endif -%} + +{% endif %}{# !project.use_mappings -#} + +{% if project.bridge_group is not none -%} +# Bridge files +# ---------------------------------------- + +{% for bridge in project.bridge_group.products -%} +{% if bridge.bridge_type == "sssom" -%} +$(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt{% for source in bridge.sources %} \ + $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} \ + | $(BRIDGEDIR) all_robot_plugins + $(ROBOT) sssom:inject {% for source in bridge.sources %}--sssom $(MAPPINGDIR)/{{ source }}.sssom.tsv \ + {% endfor %}--ruleset $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ + --create \ + annotate --ontology-iri $(ONTBASE)/bridges/{{ bridge.id }}.owl \ + --version-iri $(ONTBASE)/releases/$(VERSION)/bridges/{{ bridge.id }}.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert -f ofn -o $@ + +{% elif bridge.bridge_type == "custom" -%} +$(BRIDGEDIR)/{{ bridge.id }}.owl: + @echo "ERROR: You have configured {{ bridge.id }} as a custom bridge;" + @echo " This rule needs to be overwritten in {{ project.id}}.Makefile!" + @false + +{% endif -%} +{% endfor -%} +{% endif %}{# !project.bridge_group is not none -#} +{# #} +{% endif -%}{# !project.use_mappings or project.bridge_group is not none -#} + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -1215,100 +1327,6 @@ dosdp_validation: endif {% endif -%} -{% if project.use_mappings -%} -# ---------------------------------------- -# SSSOM Mapping Files -# ---------------------------------------- - -MAPPING_TESTER = sssom validate -SSSOMPY = sssom - -validate-sssom-%: - tsvalid $(MAPPINGDIR)/$*.sssom.tsv --comment "#" - $(MAPPING_TESTER) $(MAPPINGDIR)/$*.sssom.tsv - -validate_mappings: - $(MAKE_FAST) $(foreach n,$(MAPPINGS),validate-sssom-$(n)) - -normalize-sssom-%: - sssom-cli --output $(MAPPINGDIR)/$*.sssom.tsv $(MAPPINGDIR)/$*.sssom.tsv - -normalize_mappings: - $(MAKE_FAST) $(foreach n,$(MAPPINGS),normalize-sssom-$(n)) - -{% if project.sssom_mappingset_group is not none -%} -{% for mapping in project.sssom_mappingset_group.products -%} -{% if mapping.maintenance == "extract" -%} -{% if project.sssom_mappingset_group.mapping_extractor == "sssom-py" -%} -$(TMPDIR)/{{ mapping.id }}.obographs.json: {{ mapping.source_file }} - $(ROBOT) annotate --input $< \ - --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false --format json --output $@ - -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: $(TMPDIR)/{{ mapping.id }}.obographs.json - $(SSSOMPY) parse $< -I obographs-json {{ mapping.sssom_tool_options }} -o $@ - -{% elif project.sssom_mappingset_group.mapping_extractor == "sssom-java" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: {{ mapping.source_file }} | all_robot_plugins - $(ROBOT) sssom:xref-extract --input $< --replace \ - --ignore-treat-xrefs --all-xrefs \ - --mapping-file $@ - -{% endif -%} -{% elif mapping.maintenance == "manual" -%} -# This mappingset is manually curated, so we only check that the file actually exists. -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: - test -f $@ - -{% elif mapping.maintenance == "merged" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% for source in mapping.source_mappings %} $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} - sssom-cli --output $@ $^ - -{% elif mapping.maintenance == "mirror" -%} -ifeq ($(MIR),true) -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: - wget "{{ mapping.mirror_from }}" -O $@ -endif - -{% elif mapping.maintenance == "custom" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: - @echo "ERROR: You have configured {{ mapping.id }} as a custom mapping set;" - @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" - @false - -{% endif -%} -{% endfor -%} -{% endif -%} - -{% endif %}{# !project.use_mappings -#} - -{% if project.bridge_group is not none -%} -# ---------------------------------------- -# Bridge files -# ---------------------------------------- - -{% for bridge in project.bridge_group.products -%} -{% if bridge.bridge_type == "sssom" -%} -$(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt{% for source in bridge.sources %} \ - $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} \ - | $(BRIDGEDIR) all_robot_plugins - $(ROBOT) sssom:inject {% for source in bridge.sources %}--sssom $(MAPPINGDIR)/{{ source }}.sssom.tsv \ - {% endfor %}--ruleset $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ - --create \ - annotate --ontology-iri $(ONTBASE)/bridges/{{ bridge.id }}.owl \ - --version-iri $(ONTBASE)/releases/$(VERSION)/bridges/{{ bridge.id }}.owl \ - --annotation owl:versionInfo $(VERSION) \ - convert -f ofn -o $@ - -{% elif bridge.bridge_type == "custom" -%} -$(BRIDGEDIR)/{{ bridge.id }}.owl: - @echo "ERROR: You have configured {{ bridge.id }} as a custom bridge;" - @echo " This rule needs to be overwritten in {{ project.id}}.Makefile!" - @false - -{% endif -%} -{% endfor -%} -{% endif %}{# !project.bridge_group is not none -#} {% if project.use_translations -%} # ---------------------------------------- From 4551b5adaaba627ff35edc7679222430a284f0ac Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sat, 15 Aug 2026 18:03:39 +0100 Subject: [PATCH 08/25] Move the ORCIDIO section. The ORCIDIO module is akin to a component, so we move it to just after the components section. --- .../templates/src/ontology/Makefile.jinja2 | 79 ++++++++++--------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index d196ec0..810838c 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -805,6 +805,47 @@ no-mirror-recreate-%: {% endif %}{# ! project.components is not none -#} +{% if project.orcidio_support -%} +# ---------------------------------------- +# Special components +# ---------------------------------------- + +# ORCIDIO module +# ---------------------------------------- + +# By default, the ORCIDIO module is seeded from ORCID references in ALL +# IRI-valued annotations in the ontology, regardless of the annotation +# property. To seed only from annotations using specific properties, +# override the following variable with the desired list of properties. +ORCIDIO_PROPERTIES = + +ifeq ($(IMP),true) +$(IMPORTDIR)/orcidio_import.owl: $(SRCMERGED) $(MIRRORDIR)/orcidio.owl | all_robot_plugins + $(ROBOT) odk:extract-orcids --input $(SRCMERGED) \ + --orcid-file $(MIRRORDIR)/orcidio.owl \ + $(foreach p, $(ORCIDIO_PROPERTIES), --property $(p)) \ + $(ANNOTATE_CONVERT_FILE) + +endif + +ifeq ($(MIR),true) +.PHONY: download-mirror-orcidio +download-mirror-orcidio: | $(TMPDIR) $(MIRRORDIR) + @odk-helper download --output $(TMPDIR)/$@.owl \ + --reference $(MIRRORDIR)/orcidio.owl{% if project.import_group is defined %} \ + --max-retry {{ project.import_group.mirror_retry_download }}{% endif %} \ + https://w3id.org/orcidio/orcidio.owl + +$(MIRRORDIR)/orcidio.owl: download-mirror-orcidio + @if [ -f $(TMPDIR)/download-mirror-orcidio.owl ]; then \ + cp $(TMPDIR)/download-mirror-orcidio.owl $@ ; \ + fi + +endif + + +{% endif %}{# !project.orcidio_support -#} + {% if project.use_mappings or project.bridge_group is not none -%} # ---------------------------------------- # Mappings and bridges @@ -1085,44 +1126,6 @@ $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ -{% if project.orcidio_support -%} -# ---------------------------------------- -# ORCIDIO module -# ---------------------------------------- - -# By default, the ORCIDIO module is seeded from ORCID references in ALL -# IRI-valued annotations in the ontology, regardless of the annotation -# property. To seed only from annotations using specific properties, -# override the following variable with the desired list of properties. -ORCIDIO_PROPERTIES = - -ifeq ($(IMP),true) -$(IMPORTDIR)/orcidio_import.owl: $(SRCMERGED) $(MIRRORDIR)/orcidio.owl | all_robot_plugins - $(ROBOT) odk:extract-orcids --input $(SRCMERGED) \ - --orcid-file $(MIRRORDIR)/orcidio.owl \ - $(foreach p, $(ORCIDIO_PROPERTIES),--property $(p)) \ - $(ANNOTATE_CONVERT_FILE) - -endif - -ifeq ($(MIR),true) -.PHONY: download-mirror-orcidio -download-mirror-orcidio: | $(TMPDIR) $(MIRRORDIR) - @odk-helper download --output $(TMPDIR)/$@.owl \ - --reference $(MIRRORDIR)/orcidio.owl{% if project.import_group is defined %} \ - --max-retry {{ project.import_group.mirror_retry_download }}{% endif %} \ - https://w3id.org/orcidio/orcidio.owl - -$(MIRRORDIR)/orcidio.owl: download-mirror-orcidio - @if [ -f $(TMPDIR)/download-mirror-orcidio.owl ]; then \ - cp $(TMPDIR)/download-mirror-orcidio.owl $@ ; \ - fi - -endif - -{% endif -%} - - {% if project.subset_group is defined %} # ---------------------------------------- From 95aadceb84da2abda42ba16ca3a3855fab184060 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sat, 15 Aug 2026 21:22:14 +0100 Subject: [PATCH 09/25] Move and re-format the DOSDP section. Move the section about DOSDP patterns closer to the sections about components (same rationale as for the SSSOM or ORCIDIO sections: this is a section about something that contributes to the contents of the ontology, like a component module). As part of the re-formatting: * we update the downloading of external patterns so that it uses the same odk-helper-based logic as the downloading of all other remote resources; * we replace `$(patsubst)` by `$(basename)` to extract the base name of DOSDP pattern files (simpler and makes the intention clearer). --- .../templates/src/ontology/Makefile.jinja2 | 371 ++++++++++-------- 1 file changed, 213 insertions(+), 158 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 810838c..566be88 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -958,6 +958,219 @@ $(BRIDGEDIR)/{{ bridge.id }}.owl: {# #} {% endif -%}{# !project.use_mappings or project.bridge_group is not none -#} +{% if project.use_dosdps -%} +# ---------------------------------------- +# DOSDP templates/patterns +# ---------------------------------------- + +PATTERN_TESTER = dosdp validate -i +DOSDPT = dosdp-tools +PATTERN_RELEASE_FILES = $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl + +ALL_PATTERN_FILES = $(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml) +ALL_PATTERN_NAMES = $(strip $(basename $(notdir $(ALL_PATTERN_FILES)))) + +PATTERN_CLEAN_FILES = $(PATTERNDIR)/all_pattern_terms.txt \ + $(DOSDP_OWL_FILES_DEFAULT) $(DOSDP_TERM_FILES_DEFAULT) + {%- if project.pattern_pipelines_group is defined -%} + {% for pipeline in project.pattern_pipelines_group.products %} \ + $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}) + {%- endfor %}{% endif %} + +EXTERNAL_PATTERN_URLS = $(strip $(file <$(PATTERNDIR)/dosdp-patterns/external.txt)) +EXTERNAL_PATTERN_FILES = $(notdir $(EXTERNAL_PATTERN_URLS)) + +.PHONY: pattern_clean +pattern_clean: + rm -f $(PATTERN_CLEAN_FILES) + +ifeq ($(PAT),true) + +.PHONY: patterns dosdp +patterns dosdp: dosdp_validation $(PATTERNDIR)/pattern.owl $(PATTERNDIR)/definitions.owl + +.PHONY: pattern_schema_checks +pattern_schema_checks dosdp_validation: $(TMPDIR)/stamp-pattern-schema-checks + +$(TMPDIR)/stamp-pattern-schema-checks: $(ALL_PATTERN_FILES) | $(TMPDIR) + $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns + touch $@ + +.PHONY: download_patterns +download_patterns: | $(TMPDIR) + mkdir -p $(TMPDIR)/dosdp + $(foreach url, $(EXTERNAL_PATTERN_URLS), \ + odk-helper download --output $(TMPDIR)/dosdp/$(notdir $(url)) \ + --reference $(PATTERNDIR)/dosdp-patterns/$(notdir $(url)) \ + $(url);) + +.PHONY: update_patterns +update_patterns: download_patterns + @for f in $$(find $(TMPDIR) -type f -path '$(TMPDIR)/dosdp/*.yaml') ; do \ + cp $$f $(PATTERNDIR)/dosdp-patterns ; + done + +$(PATTERNDIR)/dosdp-patterns/%.yaml: download_patterns + @if [ -f $(TMPDIR)/dosdp/$*.yaml ]; then \ + cp $(TMPDIR)/dosdp/$*.yaml $@ ; \ + fi + +# DOSDP pipelines +# ---------------------------------------- +# Each pipeline gets its own directory structure. + +# DOSDP default pipeline +DOSDP_TSV_FILES_DEFAULT = $(wildcard $(PATTERNDIR)/data/default/*.tsv) +DOSDP_PATTERN_NAMES_DEFAULT = $(strip $(basename $(notdir $(DOSDP_TSV_FILES_DEFAULT)))) +DOSDP_OWL_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).ofn) +DOSDP_TERM_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).txt) +DOSDP_YAML_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) + +$(DOSDP_OWL_FILES_DEFAULT): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(ALL_PATTERN_FILES) + if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] ; then \ + $(DOSDPT) generate --catalog=$(CATALOG) \ + --infile=$(PATTERNDIR)/data/default/ \ + --template=$(PATTERNDIR)/dosdp-patterns \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ + --ontology=$< {% if project.dosdp_tools_options %}\ + {{ project.dosdp_tools_options }} {% endif %}\ + --outfile=$(PATTERNDIR)/data/default ; \ + fi + +.PHONY: dosdp-docs-default +dosdp-docs-default: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(DOSDP_YAML_FILES_DEFAULT) + mkdir -p $(DOCSDIR)/patterns/default + $(DOSDPT) docs {% if project.dosdp_tools_options %}{{ project.dosdp_tools_options }} \ + {% endif %}--catalog=$(CATALOG) \ + --ontology=$< \ + --infile=$(PATTERNDIR)/data/default \ + --template=$(PATTERNDIR)/dosdp-patterns \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ + --outfile=$(DOCSDIR)/patterns/default{% if project.repo_url %} \ + --data-location-prefix={{ project.repo_url }}/src/patterns/data/default{% elif project.github_org and project.repo %} \ + --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/default{% endif %} + +{% if project.pattern_pipelines_group is defined -%} +{% for pipeline in project.pattern_pipelines_group.products -%} +# DOSDP {{ pipeline.id }} pipeline +{% set PID = pipeline.id.upper() -%} + +DOSDP_TSV_FILES_{{ PID }} = $(wildcard $(PATTERNDIR)/data/{{ pipeline.id }}/*.tsv) +DOSDP_PATTERN_NAMES_{{ PID }} = $(strip $(basename $(notdir $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }})))) +DOSDP_OWL_FILES_{{ PID }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ PID }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).ofn) +DOSDP_TERM_FILES_{{ PID }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ PID }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).txt) +DOSDP_YAML_FILES_{{ PID }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ PID }}), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) + +$(DOSDP_OWL_FILES_{{ PID }}): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ PID }}) $(ALL_PATTERN_FILES) + if [ "${DOSDP_PATTERN_NAMES_{{ PID }}}" ] ; then \ + $(DOSDPT) generate --catalog=$(CATALOG) \ + --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ + --template=$(PATTERNDIR)/dosdp-patterns/ \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ PID }})" \ + --ontology=$< {% if pipeline.dosdp_tools_options %}\ + {{ pipeline.dosdp_tools_options }} {% endif %}\ + --outfile=$(PATTERNDIR)/data/{{ pipeline.id }}; \ + fi + +.PHONY: dosdp-docs-{{ pipeline.id }} +dosdp-docs-{{ pipeline.id }}: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ PID }}) $(DOSDP_YAML_FILES_{{ PID }}) + mkdir -p $(DOCSDIR)/patterns/{{ pipeline.id }} + $(DOSDPT) docs {% if pipeline.dosdp_tools_options %}{{ pipeline.dosdp_tools_options }} \ + {% endif %}--catalog=$(CATALOG) \ + --ontology=$< \ + --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ + --template=$(PATTERNDIR)/dosdp-patterns \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ PID }})" \ + --outfile=$(DOCSDIR)/patterns/{{ pipeline.id }}{% if project.repo_url %} \ + --data-location-prefix={{ project.repo_url }}/src/patterns/data/{{ pipeline.id }}{% elif project.github_org and project.repo %} \ + --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/{{ pipeline.id }}{% endif %} + +{% endfor -%} +{% endif %}{# !project.pattern_pipelines_group is defined -#} + +# Template seeds +# ---------------------------------------- + +$(PATTERNDIR)/data/default/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/default/%.tsv + $(DOSDPT) terms --infile=$(PATTERNDIR)/data/default/$*.tsv --template $< \ + --obo-prefixes=true --outfile=$@ + +{% if project.pattern_pipelines_group is defined -%} +{% for pipeline in project.pattern_pipelines_group.products -%} +$(PATTERNDIR)/data/{{ pipeline.id }}/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/{{ pipeline.id }}/%.tsv + $(DOSDPT) terms --infile=$(PATTERNDIR)/data/{{ pipeline.id }}/$*.tsv --template $< \ + --obo-prefixes=true --outfile=$@ + +{% endfor -%} +{% if project.pattern_pipelines_group.matches is iterable -%} +{% for matches in project.pattern_pipelines_group.matches -%} +dosdp-matches-{{ matches.id }}: {{ matches.ontology }} $(ALL_PATTERN_FILES) + $(DOSDPT) query {% if matches.dosdp_tools_options %}{{ matches.dosdp_tools_options }} \ + {% endif %}--ontology=$< \ + --catalog=$(CATALOG) --reasoner=elk \ + --batch-patterns="$(ALL_PATTERN_NAMES)" \ + --template="$(PATTERNDIR)/dosdp-patterns" \ + --outfile="$(PATTERNDIR)/data/{{ matches.id }}/" + +{% endfor -%} +{% endif -%} +{% endif -%} + +# Generating the seed file from all the TSVs. +$(TMPDIR)/all_pattern_terms.txt: $(DOSDP_TERM_FILES_DEFAULT) \ + {% if project.pattern_pipelines_group is defined -%} + {%- for pipeline in project.pattern_pipelines_group.products -%} + {# #}$(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}) \ + {% endfor -%} + {%- endif -%} + {# #}$(TMPDIR)/pattern_owl_seed.txt + cat $^ | sort | uniq > $@ + +$(TMPDIR)/pattern_owl_seed.txt: $(PATTERNDIR)/pattern.owl + $(ROBOT) query --input $< --use-graphs true --format csv \ + --query $(SPARQLDIR)/terms.sparql $@ + +# Pattern pipeline main targets: the generated OWL files +# ---------------------------------------- + +# An ontology of all DOSDP patterns +$(PATTERNDIR)/pattern.owl: $(ALL_PATTERN_FILES) + $(DOSDPT) prototype --template=$(PATTERNDIR)/dosdp-patterns \ + --obo-prefixes=true --outfile=$@ + +# The component containing all pattern-derived axioms +$(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) + {%- if project.pattern_pipelines_group is defined -%} + {%- for pipeline in project.pattern_pipelines_group.products %} \ + $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) + {%- endfor -%} + {%- endif %} + if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] + {%- if project.pattern_pipelines_group is defined -%} + {%- for pipeline in project.pattern_pipelines_group.products %} \ + || [ "${DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}}" ] + {%- endfor -%} + {%- endif %} ; then \ + $(ROBOT) merge $(addprefix --input , $^) \ + annotate --ontology-iri $(ONTBASE)/patterns/definitions.owl \ + --version-iri $(ONTBASE)/releases/$(TODAY)/patterns/definitions.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ ; \ + fi + +else # PAT=false +# Even if pattern generation is disabled, we still extract a seed from definitions.owl +$(TMPDIR)/all_pattern_terms.txt: $(PATTERNDIR)/definitions.owl + $(ROBOT) query --intput $< --use-graphs true --format csv \ + --query $(SPARQLDIR)/terms.sparql $@ + +# Dummy target to allow "make test" to skip DOSDP validation under PAT=false +dosdp_validation: +endif + + +{% endif %}{# !project.use_dosdps -#} + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -1172,164 +1385,6 @@ ifneq ($(SPARQL_EXPORTS_ARGS),) $(ROBOT) query -f tsv --use-graphs true -i $< $(SPARQL_EXPORTS_ARGS) endif -{% if project.use_dosdps -%} -# ---------------------------------------- -# DOSDP Templates/Patterns -# ---------------------------------------- - -PATTERN_TESTER = dosdp validate -i -DOSDPT = dosdp-tools -PATTERN_RELEASE_FILES = $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl - -ALL_PATTERN_FILES=$(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml) -ALL_PATTERN_NAMES=$(strip $(patsubst %.yaml,%, $(notdir $(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml)))) - -PATTERN_CLEAN_FILES=../patterns/all_pattern_terms.txt \ - $(DOSDP_OWL_FILES_DEFAULT) $(DOSDP_TERM_FILES_DEFAULT){% if project.pattern_pipelines_group is defined -%} - {% for pipeline in project.pattern_pipelines_group.products %} \ - $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}){% endfor %}{% endif %} - -.PHONY: pattern_clean -pattern_clean: - rm -f $(PATTERN_CLEAN_FILES) - -ifeq ($(PAT),true) - -.PHONY: patterns -patterns dosdp: - echo "Validating all DOSDP templates" - $(MAKE) dosdp_validation - echo "Building $(PATTERNDIR)/definitions.owl" - $(MAKE) $(PATTERNDIR)/pattern.owl $(PATTERNDIR)/definitions.owl - -# DOSDP Template Validation - -$(TMPDIR)/pattern_schema_checks: $(ALL_PATTERN_FILES) | $(TMPDIR) - $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns/ && touch $@ - -.PHONY: pattern_schema_checks -pattern_schema_checks dosdp_validation: $(TMPDIR)/pattern_schema_checks - -.PHONY: update_patterns -update_patterns: download_patterns - if [ -n "$$(find $(TMPDIR) -type f -path '$(TMPDIR)/dosdp/*.yaml')" ]; then cp -r $(TMPDIR)/dosdp/*.yaml $(PATTERNDIR)/dosdp-patterns; fi - -# This command is a workaround for the absence of -N and -i in wget of alpine (the one ODK depend on now). -# It downloads all patterns specified in external.txt -.PHONY: download_patterns -download_patterns: - rm -f $(TMPDIR)/dosdp/*.yaml.1 || true - if [ -s $(PATTERNDIR)/dosdp-patterns/external.txt ]; then wget -i $(PATTERNDIR)/dosdp-patterns/external.txt --backups=1 -P $(TMPDIR)/dosdp; fi - rm -f $(TMPDIR)/dosdp/*.yaml.1 || true - -$(PATTERNDIR)/dospd-patterns/%.yml: download_patterns - if cmp -s $(TMPDIR)/dosdp-$*.yml $@ ; then echo "DOSDP templates identical."; else echo "DOSDP templates different, updating." &&\ - cp $(TMPDIR)/dosdp-$*.yml $@; fi - - -# DOSDP Template: Pipelines -# Each pipeline gets its own directory structure - -# DOSDP default pipeline - -DOSDP_TSV_FILES_DEFAULT = $(wildcard $(PATTERNDIR)/data/default/*.tsv) -DOSDP_PATTERN_NAMES_DEFAULT = $(strip $(patsubst %.tsv, %, $(notdir $(DOSDP_TSV_FILES_DEFAULT)))) -DOSDP_OWL_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).ofn) -DOSDP_TERM_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).txt) -DOSDP_YAML_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) - -$(DOSDP_OWL_FILES_DEFAULT): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(ALL_PATTERN_FILES) - if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ]; then $(DOSDPT) generate --catalog=$(CATALOG) \ - --infile=$(PATTERNDIR)/data/default/ --template=$(PATTERNDIR)/dosdp-patterns --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ - --ontology=$< {{ project.dosdp_tools_options }} --outfile=$(PATTERNDIR)/data/default; fi - -.PHONY: dosdp-docs-default -dosdp-docs-default: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(DOSDP_YAML_FILES_DEFAULT) - mkdir -p $(DOCSDIR)/patterns/default - $(DOSDPT) docs {{ project.dosdp_tools_options }} --catalog=$(CATALOG) \ - --ontology=$< \ - --infile=$(PATTERNDIR)/data/default \ - --template=$(PATTERNDIR)/dosdp-patterns \ - --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ - --outfile=$(DOCSDIR)/patterns/default{% if project.repo_url %} \ - --data-location-prefix={{ project.repo_url }}/src/patterns/data/default{% elif project.github_org and project.repo %} \ - --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/default{% endif %} - -{% if project.pattern_pipelines_group is defined %} -{% for pipeline in project.pattern_pipelines_group.products %} -# DOSDP {{ pipeline.id }} pipeline - -DOSDP_TSV_FILES_{{ pipeline.id.upper() }} = $(wildcard $(PATTERNDIR)/data/{{ pipeline.id }}/*.tsv) -DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }} = $(strip $(patsubst %.tsv, %, $(notdir $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }})))) -DOSDP_OWL_FILES_{{ pipeline.id.upper() }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).ofn) -DOSDP_TERM_FILES_{{ pipeline.id.upper() }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).txt) -DOSDP_YAML_FILES_{{ pipeline.id.upper() }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) - -$(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }}) $(ALL_PATTERN_FILES) - if [ "${DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}}" ]; then $(DOSDPT) generate --catalog=$(CATALOG) \ - --infile=$(PATTERNDIR)/data/{{ pipeline.id }} --template=$(PATTERNDIR)/dosdp-patterns/ --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }})" \ - --ontology=$< {{ pipeline.dosdp_tools_options }} --outfile=$(PATTERNDIR)/data/{{ pipeline.id }}; fi - -.PHONY: dosdp-docs-{{ pipeline.id }} -dosdp-docs-{{ pipeline.id }}: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }}) $(DOSDP_YAML_FILES_{{ pipeline.id.upper() }}) - mkdir -p $(DOCSDIR)/patterns/{{ pipeline.id }} - $(DOSDPT) docs {{ pipeline.dosdp_tools_options }} --catalog=$(CATALOG) \ - --ontology=$< \ - --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ - --template=$(PATTERNDIR)/dosdp-patterns \ - --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }})" \ - --outfile=$(DOCSDIR)/patterns/{{ pipeline.id }}{% if project.repo_url %} \ - --data-location-prefix={{ project.repo_url }}/src/patterns/data/{{ pipeline.id }}{% elif project.github_org and project.repo %} \ - --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/{{ pipeline.id }}{% endif %} -{% endfor -%} -{% endif -%} - -# Generate template file seeds - -## Generate template file seeds -$(PATTERNDIR)/data/default/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/default/%.tsv - $(DOSDPT) terms --infile=$(word 2, $^) --template=$< --obo-prefixes=true --outfile=$@ - -{% if project.pattern_pipelines_group is defined -%} -{% for pipeline in project.pattern_pipelines_group.products %} -$(PATTERNDIR)/data/{{ pipeline.id }}/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/{{ pipeline.id }}/%.tsv - $(DOSDPT) terms --infile=$(word 2, $^) --template=$< --obo-prefixes=true --outfile=$@ -{% endfor %} -{% if project.pattern_pipelines_group.matches is iterable -%}{% for matches in project.pattern_pipelines_group.matches %} -dosdp-matches-{{ matches.id }}: {{ matches.ontology }} $(ALL_PATTERN_FILES) - $(DOSDPT) query --ontology=$< --catalog=$(CATALOG) --reasoner=elk {{ matches.dosdp_tools_options }} \ - --batch-patterns="$(ALL_PATTERN_NAMES)" --template="$(PATTERNDIR)/dosdp-patterns" --outfile="$(PATTERNDIR)/data/{{ matches.id }}/" -{% endfor %}{% endif -%} -{% endif -%} - -# Generating the seed file from all the TSVs. -$(TMPDIR)/all_pattern_terms.txt: $(DOSDP_TERM_FILES_DEFAULT) {% if project.pattern_pipelines_group is defined %} {% for pipeline in project.pattern_pipelines_group.products %} $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}){% endfor %}{% endif %} $(TMPDIR)/pattern_owl_seed.txt - cat $^ | sort | uniq > $@ - -$(TMPDIR)/pattern_owl_seed.txt: $(PATTERNDIR)/pattern.owl - $(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/terms.sparql $@ - -# Pattern pipeline main targets: the generated OWL files - -# Create pattern.owl, an ontology of all DOSDP patterns -$(PATTERNDIR)/pattern.owl: $(ALL_PATTERN_FILES) - $(DOSDPT) prototype --obo-prefixes true --template=$(PATTERNDIR)/dosdp-patterns --outfile=$@ - -# Generating the individual pattern modules and merging them into definitions.owl -$(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) {% if project.pattern_pipelines_group is defined %} {% for pipeline in project.pattern_pipelines_group.products %} $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}){% endfor %}{% endif %} - if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] {% if project.pattern_pipelines_group is defined %} {% for pipeline in project.pattern_pipelines_group.products %} || [ "${DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}}" ]{% endfor %}{% endif %} && [ $(PAT) = true ]; then $(ROBOT) merge $(addprefix -i , $^) \ - annotate --ontology-iri $(ONTBASE)/patterns/definitions.owl --version-iri $(ONTBASE)/releases/$(TODAY)/patterns/definitions.owl \ - --annotation owl:versionInfo $(VERSION) -o definitions.ofn && mv definitions.ofn $@; fi - -else # PAT=false -# Even if pattern generation is disabled, we still extract a seed from definitions.owl -$(TMPDIR)/all_pattern_terms.txt: $(PATTERNDIR)/definitions.owl - $(ROBOT) query --use-graphs true -f csv -i $< --query $(SPARQLDIR)/terms.sparql $@ - -dosdp_validation: -endif - -{% endif -%} {% if project.use_translations -%} # ---------------------------------------- From 60a51d0c0fde5e505fba64fa1d611fad936d3d03 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sat, 15 Aug 2026 22:51:43 +0100 Subject: [PATCH 10/25] Move and re-format the subsets section. Move the subsets section before the release section (release should come last, after all the rules needed to generate the contents of the ontology) and re-format it. The re-formatting notably involves removing stuff like this: $(ROBOT) [...] --output $@.tmp && mv $@.tmp $@ --- .../templates/src/ontology/Makefile.jinja2 | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 566be88..b2881a9 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1171,6 +1171,41 @@ endif {% endif %}{# !project.use_dosdps -#} +{% if project.subset_group is defined -%} +# ---------------------------------------- +# Subsets +# ---------------------------------------- + +$(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl + $(ROBOT) export --input $< --include classes \ + --header "ID [IRI]|LABEL" --format tsv --export $@ +.PRECIOUS: $(SUBSETDIR)/%.tsv + +$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins + $(ROBOT) odk:subset --input $< --subset $* --fill-gaps true \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) --output $@ +.PRECIOUS: $(SUBSETDIR)/%.owl + +{% if 'obo' in project.export_formats -%} +$(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl + $(ROBOT) convert --input $< --check false --format obo \ + $(OBO_FORMAT_OPTIONS) --output $@ + +{% endif -%} +{% if 'ttl' in project.export_formats -%} +$(SUBSETDIR)/%.ttl: $(SUBSETDIR)/%.owl + $(ROBOT) convert --input $< --check false --format ttl --output $@ + +{% endif -%} +{% if 'json' in project.export_formats -%} +$(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl + $(ROBOT) convert --input $< --check false --format json --output $@ + +{% endif -%} +{# #} +{% endif %}{# !project.subset_group is defined -#} + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -1340,36 +1375,6 @@ $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl -{% if project.subset_group is defined %} -# ---------------------------------------- -# Subsets -# ---------------------------------------- -$(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl - $(ROBOT) export -i $< --include classes \ - --header "ID [IRI]|LABEL" --format tsv --export $@ -.PRECIOUS: $(SUBSETDIR)/%.tsv - -$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins - $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true \ - annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@ -.PRECIOUS: $(SUBSETDIR)/%.owl - -{% if 'obo' in project.export_formats %} -$(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@ -{% endif -%} -{% if 'ttl' in project.export_formats %} -$(SUBSETDIR)/%.ttl: $(SUBSETDIR)/%.owl - $(ROBOT) convert --input $< --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats %} -$(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl - $(ROBOT) convert --input $< --check false -f json -o $@.tmp.json &&\ - mv $@.tmp.json $@ -{% endif -%} - -{% endif %} - # --------------------------------------------- # Sparql queries: Table exports / Query Reports # --------------------------------------------- From 449ca54b3ec30e00dd00d1932f5340c06e3df72f Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sat, 15 Aug 2026 23:25:50 +0100 Subject: [PATCH 11/25] Group "SPARQL exports" along with "SPARQL checks". SPARQL exports are very similar to SPARQL checks, so we can group both in a single "SPARQL checks and reports" section. We also update the code for the SPARQL exports so that it no longer uses the deprecated `--select` option of `robot query`, but the `--queries` option instead (which also dispenses us from having to construct a `SPARQL_EXPORTS_ARGS` variable). --- .../templates/src/ontology/Makefile.jinja2 | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index b2881a9..763b442 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -327,7 +327,7 @@ $(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt @echo "$* profile validation completed." -# SPARQL-based checks +# SPARQL-based checks and reports # ---------------------------------------- # The SPARQL queries used by those checks live in the ../sparql directory, @@ -335,18 +335,32 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt # NAME in the list below will enable the corresponding check. {# FIXME: The default list should be computed in the seeding script, not here. -#} -SPARQL_VALIDATION_CHECKS ={% for x in project.robot.report.custom_sparql_checks|default(["owldef-self-reference", "iri-range", "label-with-iri", "multiple-replaced_by"]) %} {{ x }}{% endfor %} -SPARQL_VALIDATION_QUERIES = $(foreach name, $(SPARQL_VALIDATION_CHECKS), $(SPARQLDIR)/$(name)-violation.sparql) +SPARQL_VALIDATION_CHECKS ={% for x in project.robot.report.custom_sparql_checks|default(["owldef-self-reference", "iri-range", "label-with-iri", "multiple-replaced_by"]) %} {{ x }}{% endfor %} sparql_test:{% for x in project.robot.report.sparql_test_on|default(["edit"]) %} {% if x =="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %}{% endfor %} | $(REPORTDIR) -ifneq ($(SPARQL_VALIDATION_QUERIES),) +ifneq ($(SPARQL_VALIDATION_CHECKS),) {% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} {# #} $(ROBOT) verify --input {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %} \ - --queries $(SPARQL_VALIDATION_QUERIES) \ - --output-dir $(REPORTDIR) + --queries $(foreach name, $(SPARQL_VALIDATION_CHECKS), $(SPARQLDIR)/$(name)-violation.sparql) \ + --output-dir $(REPORTDIR) {% endfor -%} endif +# SPARQL "exports" differ from SPARQL checks in that +# (i) the files containing the queries do not have a '-violation' suffix, +# (ii) they do not trigger an error, they merely report what they found. + +{# FIXME: The default list should be computed in the seeding script, not here. -#} +SPARQL_EXPORTS ={% for x in project.robot.report.custom_sparql_exports|default(["basic-report", "class-count-by-prefix", "edges", "xrefs", "obsoletes", "synonyms"]) %} {{ x }}{% endfor %} + +.PHONY: custom_reports +custom_reports: $(EDIT_PREPROCESSED) | $(REPORTDIR) +ifneq ($(SPARQL_EXPORTS),) + $(ROBOT) query --input $< --use-graphs true \ + --queries $(foreach name, $(SPARQL_EXPORTS), $(SPARQLDIR)/$(name).sparql) \ + --format tsv --output-dir $(REPORTDIR) +endif + {% if project.import_group is defined -%} # ---------------------------------------- @@ -1375,22 +1389,6 @@ $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl -# --------------------------------------------- -# Sparql queries: Table exports / Query Reports -# --------------------------------------------- - -SPARQL_EXPORTS = {% for x in project.robot.report.custom_sparql_exports|default(['basic-report', 'class-count-by-prefix', 'edges', 'xrefs', 'obsoletes', 'synonyms']) %}{{ x }} {% endfor %} - -SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql $(REPORTDIR)/$(V).tsv) -# This combines all into one single command - -.PHONY: custom_reports -custom_reports: $(EDIT_PREPROCESSED) | $(REPORTDIR) -ifneq ($(SPARQL_EXPORTS_ARGS),) - $(ROBOT) query -f tsv --use-graphs true -i $< $(SPARQL_EXPORTS_ARGS) -endif - - {% if project.use_translations -%} # ---------------------------------------- # Babelon Translation Files From 366f5086852ce474cb193060cb126fe03a127773 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 16 Aug 2026 01:55:48 +0100 Subject: [PATCH 12/25] Move and re-format the translation section. Move the translation section before the release section and re-format it. Re-formatting notably includes removing some repeated tests "if project.babelon_translation_group is not none" to instead enclose most of the section in one single such test. Also, discovered the Jinja2 "loop" variable, much useful when having to format for loops. --- .../templates/src/ontology/Makefile.jinja2 | 196 ++++++++++-------- 1 file changed, 109 insertions(+), 87 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 763b442..0735cfd 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -912,9 +912,9 @@ $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: test -f $@ {% elif mapping.maintenance == "merged" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% if mapping.source_mappings|length >=1 %} $(MAPPINGDIR)/{{ mapping.source_mappings[0] }}.sssom.tsv{% endif %} - {%- for source in mapping.source_mappings[1:] %} \ - $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% for source in mapping.source_mappings %} $(MAPPINGDIR)/{{ source }}.sssom.tsv + {%- if not loop.last %} \ + {% endif %}{% endfor %} sssom-cli --output $@ $^ {% elif mapping.maintenance == "mirror" -%} @@ -1220,6 +1220,112 @@ $(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl {# #} {% endif %}{# !project.subset_group is defined -#} +{% if project.use_translations -%} +# ---------------------------------------- +# Babelon translation files +# ---------------------------------------- + +BABELONPY = babelon -q +{% if project.babelon_translation_group is not none -%} +TRANSLATIONS_OWL ={% for translation in project.babelon_translation_group.products %} $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.owl + {%- if translation.include_robot_template_synonyms %} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.owl{% endif %} + {%- if not loop.last %} \ + {% endif %}{% endfor %} +TRANSLATIONS_TSV ={% for translation in project.babelon_translation_group.products %} $(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv + {%- if not loop.last %} \ + {% endif %}{% endfor %} +TRANSLATION_FILES ={% if project.babelon_translation_group.release_merged_translations %} + {#- #} $(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json + {%- endif %} +TRANSLATIONS_ADAPTER = {{ project.babelon_translation_group.oak_adapter|default("pronto:$(ONT).obo") }} +TRANSLATIONS_ONTOLOGY = {{ project.babelon_translation_group.translate_ontology|default("$(ONT).obo") }} +TRANSLATE_PREDICATES ={% for predicate_id in project.babelon_translation_group.predicates|default(["IAO:0000115", "rdfs:label"], true) %} + {#- #} {{ predicate_id }}{% endfor %} + +{% for translation in project.babelon_translation_group.products -%} +{% if translation.maintenance == "mirror" -%} +$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv: + wget "{{ translation.mirror_babelon_from }}" -O $@ + +{% if translation.include_robot_template_synonyms -%} +$(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.tsv: + wget "{{ translation.mirror_synonyms_from }}" -O $@ + +{% endif -%} +{% else -%} +# This mappingset is manually curated, so we only check that the file actually exists. +$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv: + test -f $@ + +{% if translation.include_robot_template_synonyms -%} +$(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.tsv: + test -f $@ + +{% endif -%} +{% endif -%} + +$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv +{% if translation.auto_translate -%} +{# #} @if [ -z "$(OPENAI_API_KEY)" ] ; then \ + echo "ERROR: OPENAI_API_KEY must be set as as part of the make command," ; \ + echo " e.g. sh run.sh make OPENAI_API_KEY=\"sk-123\" my_command." ; \ + false ; \ + fi +{% endif -%} +{# #} $(BABELONPY) prepare-translation $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv \ + --oak-adapter $(TRANSLATIONS_ADAPTER) \ + --language-code {{ translation.language|default("en") }} \ + $(foreach n, $(TRANSLATE_PREDICATES), --field $(n)) \ + --output-source-changed $(TRANSLATIONSDIR)/{{ translation.id }}-changed.babelon.tsv \ + --output-not-translated $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv \ + --include-not-translated {{ translation.include_not_translated|default("false")|lower }} \ + --update-translation-status {{ translation.update_translation_status|default("true")|lower }} \ + --drop-unknown-columns {{ translation.drop_unknown_columns|default("true")|lower }} \ + -o $@ +{% if translation.auto_translate -%} +{# #} @echo "Warning: By default, the toolkit employs LLM-mediated translations" + @echo " using the OpenAI API. This default may change at any time." + @echo "Warning: Never store API keys or other secrets in Makefiles or" + @echo " scripts you have in version control." + export OPENAI_API_KEY="$(OPENAI_API_KEY)" && \ + $(BABELONPY) translate $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv \ + -o $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv + $(BABELONPY) merge $(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv \ + $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv \ + -o $@ +{% endif -%} +{# #} +{% endfor -%} +{% endif %}{# !project.babelon_translation_group is not none -#} + +$(TRANSLATIONSDIR)/%.synonyms.owl: $(TRANSLATIONSDIR)/%.synonyms.tsv + $(ROBOT) template --template $< \ + annotate --ontology-iri $(ONTBASE)/translations/$*.synonyms.owl \ + --version-iri $(ONTBASE)/releases/$(VERSION)/translations/$*.synonyms.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ +.PRECIOUS: $(TRANSLATIONSDIR)/%.synonyms.owl + +$(TMPDIR)/%.babelon.owl: $(TRANSLATIONSDIR)/%-preprocessed.babelon.tsv | $(TMPDIR) + $(BABELONPY) convert $< --output-format owl -o $@ + +$(TRANSLATIONSDIR)/%.babelon.owl: $(TMPDIR)/%.babelon.owl + $(ROBOT) merge --input $< \ + annotate --ontology-iri $(ONTBASE)/translations/$*.babelon.owl \ + --version-iri $(ONTBASE)/releases/$(VERSION)/translations/$*.babelon.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ +.PRECIOUS: $(TRANSLATIONSDIR)/%.babelon.owl + +$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv: $(TRANSLATIONS_TSV) + $(BABELONPY) merge $^ -o $@ + +$(TRANSLATIONSDIR)/%.babelon.json: $(TRANSLATIONSDIR)/%.babelon.tsv + $(BABELONPY) convert $< --output-format json -o $@ + + +{% endif %}{# !project.use_translations -#} + # ---------------------------------------- # Release assets # ---------------------------------------- @@ -1388,90 +1494,6 @@ $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ - -{% if project.use_translations -%} -# ---------------------------------------- -# Babelon Translation Files -# ---------------------------------------- - -BABELONPY = babelon -q -TRANSLATIONS_OWL = {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.owl {% if translation.include_robot_template_synonyms %} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.owl {% endif %}{% endfor %} -TRANSLATIONS_TSV = {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv {% endfor %} -TRANSLATION_FILES = {%- if project.babelon_translation_group is not none %}{% if project.babelon_translation_group.release_merged_translations %}$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json{% endif %}{% endif %} - -{%- if project.babelon_translation_group is not none %} - -TRANSLATIONS_ADAPTER={{ project.babelon_translation_group.oak_adapter|default('pronto:$(ONT).obo') }} -TRANSLATIONS_ONTOLOGY={{ project.babelon_translation_group.translate_ontology|default('$(ONT).obo') }} -TRANSLATE_PREDICATES={% for predicate_id in project.babelon_translation_group.predicates|default(['IAO:0000115', 'rdfs:label'], true) %}{{ predicate_id }} {% endfor %} - -{% for translation in project.babelon_translation_group.products %} -{% if translation.maintenance == "mirror" %} -$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv: - wget "{{ translation.mirror_babelon_from }}" -O $@ -{% if translation.include_robot_template_synonyms %} -$(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.tsv: - wget "{{ translation.mirror_synonyms_from }}" -O $@ -{% endif %} -{% else %} -# This mappingset is manually curated, so we only check that the file actually exists. -$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv: - test -f $@ -{% if translation.include_robot_template_synonyms %} -$(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.tsv: - test -f $@ -{% endif %} -{% endif %} - -$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv{% if translation.auto_translate %} - @if [ -z "$(OPENAI_API_KEY)" ]; then echo "OPENAI_API_KEY must be set as as part of the make command, e.g. sh run.sh make OPENAI_API_KEY=\"sk-123\" my_command" && exit 1; fi{% endif %} - $(BABELONPY) prepare-translation $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv \ - --oak-adapter $(TRANSLATIONS_ADAPTER) \ - --language-code {{ translation.language|default('en') }} \ - $(foreach n,$(TRANSLATE_PREDICATES), --field $(n)) \ - --output-source-changed $(TRANSLATIONSDIR)/{{ translation.id }}-changed.babelon.tsv \ - --output-not-translated $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv \ - --include-not-translated {{ translation.include_not_translated|default('false')|lower }} \ - --update-translation-status {{ translation.update_translation_status|default('true')|lower }} \ - --drop-unknown-columns {{ translation.drop_unknown_columns|default('true')|lower }} \ - -o $@{% if translation.auto_translate %} - echo "Warning: By default, the toolkit employs LLM-mediated translations using the OpenAI API. This default may change at any time" - echo "Warning: Never store API keys or other secrets in Makefiles or scripts you have in version control." - export OPENAI_API_KEY="$(OPENAI_API_KEY)" &&\ - $(BABELONPY) translate $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv -o $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv - $(BABELONPY) merge $(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv -o $@ - {%- endif %} - -{% endfor %} -{%- endif %} - -$(TRANSLATIONSDIR)/%.synonyms.owl: $(TRANSLATIONSDIR)/%.synonyms.tsv - $(ROBOT) template --template $< \ - annotate \ - --ontology-iri $(ONTBASE)/translations/$*.synonyms.owl \ - -V $(ONTBASE)/releases/$(VERSION)/translations/$*.synonyms.owl \ - --annotation owl:versionInfo $(VERSION) \ - convert --format ofn --output $@ -.PRECIOUS: $(TRANSLATIONSDIR)/%.synonyms.owl - -$(TRANSLATIONSDIR)/%.babelon.owl: $(TRANSLATIONSDIR)/%-preprocessed.babelon.tsv - $(BABELONPY) convert $< --output-format owl -o $@.tmp - $(ROBOT) merge -i $@.tmp \ - annotate \ - --ontology-iri $(ONTBASE)/translations/$*.babelon.owl \ - -V $(ONTBASE)/releases/$(VERSION)/translations/$*.babelon.owl \ - --annotation owl:versionInfo $(VERSION) \ - convert --format ofn --output $@ - @rm $@.tmp -.PRECIOUS: $(TRANSLATIONSDIR)/%.babelon.owl - -$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv: $(TRANSLATIONS_TSV) - $(BABELONPY) merge $^ -o $@ - -$(TRANSLATIONSDIR)/%.babelon.json: $(TRANSLATIONSDIR)/%.babelon.tsv - $(BABELONPY) convert $< --output-format json -o $@ - -{% endif -%} # ---------------------------------------- # Release artefacts: export formats # ---------------------------------------- From d8b29f03e734435e8e2bb5fb5d26875aaabe3f79 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 16 Aug 2026 15:13:19 +0100 Subject: [PATCH 13/25] Move and re-format the release artefacts section. Re-format all the rules to produce the release artefacts and move them before the release section. The re-formatting makes intensive use of Jinja's whitespace control to try making the rules as readable as possible. --- .../templates/src/ontology/Makefile.jinja2 | 596 ++++++++++++------ 1 file changed, 386 insertions(+), 210 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 0735cfd..6cc3b6b 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1326,6 +1326,392 @@ $(TRANSLATIONSDIR)/%.babelon.json: $(TRANSLATIONSDIR)/%.babelon.tsv {% endif %}{# !project.use_translations -#} +# ---------------------------------------- +# Release artefacts +# ---------------------------------------- + +SHARED_ROBOT_COMMANDS ={% if project.remove_owl_nothing %} remove --term owl:Nothing{% endif %} +{# Merge all OWL imports (both ODK import modules and components). -#} +ROBOT_RELEASE_IMPORT_MODE = {% if project.use_edit_file_imports -%} + $(ROBOT) merge --input $<{# -#} + {% if project.import_group is not none and + project.import_group.use_base_merging is false and + project.import_group.annotate_defined_by -%} + {# #} --annotate-defined-by true{# -#} + {% endif -%} + {% else -%} + $(ROBOT) remove --input $< --select imports --trim false{# -#} + {% if project.use_dosdps or project.components is not none %} \ + merge $(foreach src, $(OTHER_SRC), --input $(src)){# -#} + {% endif -%} + {% if project.import_group is not none %} \ + merge $(foreach imp, $(IMPORT_FILES), --input $(imp)){# -#} + {% endif -%} + {% endif %} +{# Merge components only. + This is for the release artefacts that start from "base". -#} +ROBOT_RELEASE_IMPORT_MODE_BASE = $(ROBOT) remove --input $< --select imports --trim false{# -#} + {% if project.use_dosdps or project.components is not none %} \ + merge $(foreach src, $(OTHER_SRC), --input $(src)){# -#} + {% endif %} + +# Release artefacts: main pipelines +# ---------------------------------------- + +{% if "basic" in project.release_artefacts or project.primary_release == "basic" -%} +KEEPRELATIONS = keeprelations.txt +{% endif -%} +{% if "basic" in project.release_artefacts + or "simple" in project.release_artefacts + or "simple-non-classified" in project.release_artefacts + or project.primary_release == "basic" + or project.primary_release == "simple" + or project.primary_release == "simple-non-classified" -%} +SIMPLESEED = $(TMPDIR)/simple_seed.txt + +$(SIMPLESEED): $(SRCMERGED) + $(ROBOT) query --input $< --format csv \ + --query $(SPARQLDIR)/simple-seed.sparql $@.1.tmp \ + --query $(SPARQLDIR)/{{ project.id }}_terms.sparql $@.2.tmp + cat $@.1.tmp $@.2.tmp | sort | uniq > $@ + echo "http://www.geneontology.org/formats/oboInOwl#SubsetProperty" >> $@ + echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ + +{% endif -%} + +# The primary release artefact. +$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl + $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) --output $@ + +{% if "base" in project.release_artefacts or project.primary_release == "base" -%} +# Base: A version of the ontology in which axioms outside of the +# ontology's namespace(s) are forcibly removed. +$(ONT)-base.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + {% if project.release_use_reasoner -%} + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default("false")|lower }} \ + {% if project.release_materialize_object_properties -%} + materialize {# -#} + {% for iri in project.release_materialize_object_properties -%} + --term {{ iri }}{# -#} + {% if not loop.last %} \ + {# -#} + {% else %} \ + {# -#} + {% endif -%} + {% endfor -%} + {% endif -%} + {% endif -%} + relax $(RELAX_OPTIONS) \ + {% if project.release_use_reasoner -%} + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + {% endif -%} + remove {# -#} + {% if project.namespaces is not none -%} + {% for iri in project.namespaces -%} + --base-iri {{ iri }}{# -#} + {% if not loop.last %} \ + {# -#} + {% else %} \ + {# -#} + {% endif -%} + {% endfor -%} + {% else -%} + --base-iri $(URIBASE)/{{ project.id.upper() }} \ + {% endif -%} + --axioms external --preserve-structure false --trim false \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ + --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% if "baselite" in project.release_artefacts or project.primary_release == "baselite" -%} +# Baselite: All the axioms as intended by the editors (no imports, no reasoning). +{# FIXME: This is undocumented and of dubious usefulness. This was previously + said in a comment here to be "the same as base", which it is absolutely not. + See https://github.com/INCATools/ontology-development-kit/issues/1102. -#} +$(ONT)-baselite.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ + $(ROBOT_SHARED_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% if "full" in project.release_artefacts or project.primary_release == "full" -%} +# Full: The full artefacts with imports merged, reasoned. +$(ONT)-full.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + {% if project.release_materialize_object_properties -%} + materialize {# -#} + {% for iri in project.release_materialize_object_properties -%} + --term {{ iri }}{# -#} + {% if not loop.last %} \ + {# -#} + {% else %} \ + {# -#} + {% endif -%} + {% endfor -%} + {% endif -%} + relax $(RELAX_OPTIONS) \ + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% if "non-classified" in project.release_artefacts or project.primary_release == "non-classified" -%} +# Non-classified: All the axioms as intended by the editors, +# including imported axioms but without reasoning. +$(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + $(ROBOT_SHARED_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% if "simple" in project.release_artefacts or project.primary_release == "simple" -%} +# Simple: A version of the ontology that is {% if not project.release_use_reasoner %}un{% endif%}reasoned +# and from which all axioms referring to an entity outside +# of the ontology's namespace(s) are forcibly removed. +$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} | all_robot_plugins + $(ROBOT_RELEASE_IMPORT_MODE) \ + {% if project.release_use_reasoner -%} + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default("false")|lower }} \ + {% endif -%} + relax $(RELAX_OPTIONS) \ + remove --axioms equivalent \ + filter --term-file $(SIMPLESEED) \ + --select "annotations ontology anonymous self" \ + --trim true --signature true \ + {% if project.release_use_reasoner -%} + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + {% endif -%} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ROBOT_SHARED_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% if "simple-non-classified" in project.release_artefacts or project.primary_release == "simple-non-classified" %} +# Simple-non-classified: Does not merge imports AND forcibly removes +# axioms outside of the ontology's namespaces(s). No reasoning. +$(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ + remove --axioms equivalent \ + {% if project.release_use_reasoner -%} + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + {% endif -%} + filter --select ontology --term-file $(SIMPLESEED) --trim false \ + $(ROBOT_SHARED_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% if "basic" in project.release_artefacts or project.primary_release == "basic" %} +# Basic: A version of the ontology that is {% if not project.release_use_reasoner %}un{% endif%}reasoned +# and from which all axioms referring to an entity outside +# of the ontology's namespace(s) are forcibly removed, as well +# as all relationships not whitelisted in KEEPRELATIONS. +$(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) $(KEEPRELATIONS) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + {% if project.release_use_reasoner -%} + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default("false")|lower }} \ + {% endif -%} + relax $(RELAX_OPTIONS) \ + remove --axioms equivalent \ + remove --axioms disjoint \ + remove --term-file $(KEEPRELATIONS) \ + --select complement --select object-properties --trim true \ + filter --term-file $(SIMPLESEED) \ + --select "annotations ontology anonymous self" \ + --trim true --signature true \ + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + $(ROBOT_SHARED_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + + +{% if "international" in project.release_artefacts or project.primary_release == "international" -%} +# International: A variant of the primary release, but with multi-language support. +$(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) + $(ROBOT) merge $(foreach src, $^, --input $(src)) \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + +{% endif -%} + +{% for r in project.release_artefacts -%} +{% if r.startswith("custom-") -%} +{{ r | replace("custom-","")}}.owl: + @echo "ERROR: You have configured a custom release artefact ($@);" + @echo " this artefact needs to be defined in {{ project.id }}.Makefile!" + @false + +{% endif -%} +{% endfor -%} + +# Release artefacts: export formats +# ---------------------------------------- + +{# FIXME: The seeding script should provide us with a list of artefacts + that include both the "primary" artefact ($(ONT)) plus all the + variants ($(ONT)-base, $(ONT)-full, etc.), so that we do not have + to duplicate all the code below (once for the primary artefact and + one when we iterate over the variants. -#} +{% if "obo" in project.export_formats -%} +$(ONT).obo: $(ONT).owl + $(ROBOT) convert --input $< --check false \ + --format obo $(OBO_FORMAT_OPTIONS) --output $@ + +{% endif -%} +{% if "ttl" in project.export_formats -%} +{# FIXME: Why are re-annotating? The release.owl product should have + been annotated already. + See https://github.com/INCATools/ontology-development-kit/issues/1366. -#} +$(ONT).ttl: $(ONT).owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format ttl --output $@ + +{% endif -%} +{% if "json" in project.export_formats -%} +$(ONT).json: $(ONT).owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format json --output $@ + +{% endif -%} +{% for r in project.release_artefacts -%} +{% if r.startswith("custom-") -%} +{% set release = r | replace("custom-","") -%} +{% else -%} +{% set release = "$(ONT)-" ~ r -%} +{% endif -%} +{% if "obo" in project.export_formats -%} +{{ release }}.obo: {{ release }}.owl + $(ROBOT) convert --input $< --check false \ + --format obo $(OBO_FORMAT_OPTIONS) --output $@ + +{% endif -%} +{% if "ttl" in project.export_formats -%} +{{ release }}.ttl: {{ release }}.owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format ttl --output $@ + +{% endif -%} +{% if "json" in project.export_formats -%} +{{ release }}.json: {{ release }}.owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format json --output $@ + +{% endif -%} +{% endfor -%} + +{% if 'db' in project.export_formats -%} + +{% if project.use_context -%} +CONTEXT_FILE_CSV = $(TMPDIR)/context.csv +$(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) + @odk-helper context2csv $< > $@ + +{% endif -%} + +%.db: %.owl{% if project.use_context %} $(CONTEXT_FILE_CSV){% endif %} + @rm -f $*.db $*-relation-graph.tsv.gz .template.db .template.db.tmp + semsql make $*.db{% if project.use_context %} -P $(CONTEXT_FILE_CSV){% endif %} + @rm -f $*-relation-graph.tsv.gz .template.db .template.db.tmp + @test -f $*.db || (echo "SQLite/SemSQL generation failed" && exit 1) + +{% endif -%} + +{% for format in project.export_formats -%} +{% if project.gzip_main -%} +$(ONT).{{ format }}.gz: $(ONT).{{ format }} + gzip -c $< > $@ + +{% endif -%} +{% endfor -%} +{% if 'owl' not in project.export_formats -%} +{% if project.gzip_main -%} +$(ONT).owl.gz: $(ONT).owl + gzip -c $< > $@ + +{% endif -%} +$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl + cp $< $@ + +{% endif -%} + +{# #} # ---------------------------------------- # Release assets # ---------------------------------------- @@ -1447,9 +1833,6 @@ check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product). # Release Management # ---------------------------------------- -{% if 'basic' in project.release_artefacts or project.primary_release == 'basic' -%} -KEEPRELATIONS=keeprelations.txt -{% endif -%} CLEANFILES=$(MAIN_FILES) $(SRCMERGED) $(EDIT_PREPROCESSED) @@ -1494,215 +1877,8 @@ $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ -# ---------------------------------------- -# Release artefacts: export formats -# ---------------------------------------- - - -{% for r in project.release_artefacts -%} -{% if r.startswith('custom-') %}{% set release = r | replace("custom-","") %}{% else %}{% set release = "$(ONT)-" ~ r %}{% endif -%} -{% if 'obo' in project.export_formats -%} -{{ release }}.obo: {{ release }}.owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@ -{% endif -%} -{% if 'ttl' in project.export_formats -%} -{{ release }}.ttl: {{ release }}.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats -%} -{{ release }}.json: {{ release }}.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f json -o $@.tmp.json &&\ - mv $@.tmp.json $@ -{% endif -%} -{% endfor -%} - -{% if 'db' in project.export_formats -%} - -{% if project.use_context -%} -CONTEXT_FILE_CSV = $(TMPDIR)/context.csv -$(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) - @odk-helper context2csv $< > $@ -{% endif -%} - -%.db: %.owl{% if project.use_context %} $(CONTEXT_FILE_CSV){% endif %} - @rm -f $*.db $*-relation-graph.tsv.gz .template.db .template.db.tmp - semsql make $*.db{% if project.use_context %} -P $(CONTEXT_FILE_CSV){% endif %} - @rm -f $*-relation-graph.tsv.gz .template.db .template.db.tmp - @test -f $*.db || (echo "SQLite/SemSQL generation failed" && exit 1) -{% endif -%} - -# ---------------------------------------- -# Release artefacts: main release artefacts -# ---------------------------------------- - -$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl - $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert -o $@.tmp.owl && mv $@.tmp.owl $@ - -{% if 'obo' in project.export_formats -%} -$(ONT).obo: $(ONT).owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@ -{% endif -%} -{% if 'ttl' in project.export_formats -%} -$(ONT).ttl: $(ONT).owl - $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats -%} -$(ONT).json: $(ONT).owl - $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f json -o $@.tmp.json &&\ - mv $@.tmp.json $@ -{% endif -%} - -# ----------------------------------------------------- -# Release artefacts: variants (base, full, simple, etc) -# ----------------------------------------------------- -SHARED_ROBOT_COMMANDS = {% if project.remove_owl_nothing -%}remove --term owl:Nothing{% endif %} - -{% if "basic" in project.release_artefacts - or "simple" in project.release_artefacts - or "simple-non-classified" in project.release_artefacts - or project.primary_release == "basic" - or project.primary_release == "simple" - or project.primary_release == "simple-non-classified" -%} -SIMPLESEED = $(TMPDIR)/simple_seed.txt - -$(SIMPLESEED): $(SRCMERGED) - $(ROBOT) query --input $< --format csv \ - --query $(SPARQLDIR)/simple-seed.sparql $@.1.tmp \ - --query $(SPARQLDIR)/{{ project.id }}_terms.sparql $@.2.tmp - cat $@.1.tmp $@.2.tmp | sort | uniq > $@ - echo "http://www.geneontology.org/formats/oboInOwl#SubsetProperty" >> $@ - echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ -{% endif -%} -{% for format in project.export_formats -%} -{% if project.gzip_main -%} -$(ONT).{{ format }}.gz: $(ONT).{{ format }} - gzip -c $< > $@.tmp && mv $@.tmp $@ -{% endif -%} -{% endfor -%} -{% if 'owl' not in project.export_formats -%} -{% if project.gzip_main -%} -$(ONT).owl.gz: $(ONT).owl - gzip -c $< > $@.tmp && mv $@.tmp $@ -{% endif -%} -$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl - cp $< $@ -{% endif -%} - -# ROBOT pipeline that merges imports, including components. -ROBOT_RELEASE_IMPORT_MODE={% if project.use_edit_file_imports -%}$(ROBOT) merge --input $< {% if project.import_group is defined and project.import_group.use_base_merging is false and project.import_group.annotate_defined_by %}--annotate-defined-by true{% endif -%}{% else -%} -$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} -merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% if project.import_group is defined -%} -merge $(patsubst %, -i %, $(IMPORT_FILES)){% endif %}{% endif %} - -# ROBOT pipeline that removes imports, then merges components. This is for release artefacts that start from "base" -ROBOT_RELEASE_IMPORT_MODE_BASE=$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} -merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %} - -{% if 'base' in project.release_artefacts or project.primary_release == 'base' -%} -# base: A version of the ontology that does not include any externally imported axioms. -$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% if project.release_materialize_object_properties is defined and project.release_materialize_object_properties %} - materialize {% for iri in project.release_materialize_object_properties %}--term {{iri}} {% endfor %} \{% endif %}{% endif %} - relax $(RELAX_OPTIONS) \{% if project.release_use_reasoner %} - reduce -r $(REASONER) $(REDUCE_OPTIONS) \{% endif %} - remove {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ - $(SHARED_ROBOT_COMMANDS) \ - annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ - --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'baselite' in project.release_artefacts or project.primary_release == 'baselite' -%} -# baselite: All the axioms as they are editted by the editors, excluding reasoning. This is currently the same as "base". Only to be used experimentally (may disappear in future ODK releases). -$(ONT)-baselite.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) - $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'full' in project.release_artefacts or project.primary_release == 'full' -%} -# Full: The full artefacts with imports merged, reasoned. -$(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% if project.release_materialize_object_properties is defined and project.release_materialize_object_properties %} - materialize {% for iri in project.release_materialize_object_properties %}--term {{iri}} {% endfor %} \{% endif %} - relax $(RELAX_OPTIONS) \ - reduce -r $(REASONER) $(REDUCE_OPTIONS) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'non-classified' in project.release_artefacts or project.primary_release == 'non-classified' -%} -# foo-non-classified: (edit->imports-merged) -$(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'simple' in project.release_artefacts or project.primary_release == 'simple' -%} -# foo-simple: (edit->reason,relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest") -# drop every axiom: filter --term-file keep_terms.txt --trim true -# remove --select imports --trim false -$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) | all_robot_plugins - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% endif %} - relax $(RELAX_OPTIONS) \ - remove --axioms equivalent \ - filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \{% if project.release_use_reasoner %} - reduce -r $(REASONER) $(REDUCE_OPTIONS) \{% endif %} - odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'simple-non-classified' in project.release_artefacts or project.primary_release == 'simple-non-classified' %} -# foo-simple-non-classified (edit->relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest") - aka the HPO use case, no reason. -# Should this be the non-classified ontology with the drop foreign axiom filter? -# Consider adding remove --term "http://www.geneontology.org/formats/oboInOwl#hasOBONamespace" -$(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - remove --axioms equivalent \{% if project.release_use_reasoner %} - reduce -r $(REASONER) $(REDUCE_OPTIONS) \{% endif %} - filter --select ontology --term-file $(SIMPLESEED) --trim false \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'international' in project.release_artefacts or project.primary_release == 'international' -%} -# international: A variant of the primary_release, but with multi-language support. -$(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) - $(ROBOT) merge $(patsubst %, -i %, $^) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} - -{% if 'basic' in project.release_artefacts or project.primary_release == 'basic' %} -# foo-basic: A version of -simple containing only relationships using relations on a configurable whitelist (default = BFO:0000050 (?)). -# See above (David comment) for explanation. -# removes any axioms that contains one of the ops that not in the whitelist file -$(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(KEEPRELATIONS) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% endif %} - relax $(RELAX_OPTIONS) \ - remove --axioms equivalent \ - remove --axioms disjoint \ - remove --term-file $(KEEPRELATIONS) --select complement --select object-properties --trim true \ - filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \ - reduce -r $(REASONER) $(REDUCE_OPTIONS) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ -{% endif -%} -{% for r in project.release_artefacts %} -{%- if r.startswith('custom-') %} -{{ r | replace("custom-","")}}.owl: - echo "ERROR: You have configured a custom release artefact ($@); this release artefact needs to be define in {{ project.id }}.Makefile!" && false -{% endif -%} -{% endfor -%} # ---------------------------------------- # Debugging Tools From 8826846124ddafc9aadcb5ad15275a4c351149cb Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 17 Aug 2026 20:52:31 +0100 Subject: [PATCH 14/25] Re-arrange and re-format the release section. Move most of the assets definitions to their corresponding sections (i.e. mapping assets in the mappings section, subset assets in the subsets section, etc.) I think it is better to have the list of assets defined close to the place where the rules that build them are also located. The "assets" subsection in the release section can then focus on assembling the complete list of assets. (We only need an exception for imports, because IMPORT_FILES depend both on the imports section _and_ on the ORCIDIO section.) Also move the code that creates the "release diff" to a temporary place in the "editor utilities" section. Despite the "release diff", it does not really has anything to do with the release process (and it is not part of it). --- .../templates/src/ontology/Makefile.jinja2 | 292 +++++++++--------- 1 file changed, 146 insertions(+), 146 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 6cc3b6b..a307a64 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -869,8 +869,15 @@ endif # SSSOM mappings sets # ---------------------------------------- -MAPPING_TESTER = sssom validate -SSSOMPY = sssom +MAPPINGS ={% for m in project.sssom_mappingset_group.products %} {{ m.id }}{% endfor %} +RELEASED_MAPPINGS ={% for m in project.sssom_mappingset_group.released_products %} {{ m.id }}{% endfor %} +MAPPING_FILES = $(foreach m, $(MAPPINGS), $(MAPPINGDIR)/$(m).sssom.tsv) +RELEASED_MAPPING_FILES = $(foreach m, $(RELEASED_MAPPINGS), $(MAPPINGDIR)/$(m).sssom.tsv) +MAPPING_TESTER = sssom validate +SSSOMPY = sssom + +.PHONY: all_mappings +all_mappings: $(MAPPING_FILES) validate-sssom-%: tsvalid $(MAPPINGDIR)/$*.sssom.tsv --comment "#" @@ -947,6 +954,12 @@ $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: # Bridge files # ---------------------------------------- +BRIDGES ={% for bridge in project.bridge_group.products %} {{ bridge.id }}{% endfor %} +BRIDGE_FILES = $(foreach b, $(BRIDGES), $(BRIDGEDIR)/$(b).owl) + +.PHONY: all_bridges +all_bridges: $(BRIDGE_FILES) + {% for bridge in project.bridge_group.products -%} {% if bridge.bridge_type == "sssom" -%} $(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt{% for source in bridge.sources %} \ @@ -1185,11 +1198,17 @@ endif {% endif %}{# !project.use_dosdps -#} -{% if project.subset_group is defined -%} +{% if project.subset_group is not none -%} # ---------------------------------------- # Subsets # ---------------------------------------- +SUBSETS ={% for s in project.subset_group.products %} {{ s.id }}{% endfor %} +SUBSET_FILES = $(foreach s, $(SUBSET), $(foreach f, $(sort $(FORMATS) tsv), $(SUBSETDIR)/$(s).$(f))) + +.PHONY: all_subsets +all_subsets: $(SUBSET_FILES) + $(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl $(ROBOT) export --input $< --include classes \ --header "ID [IRI]|LABEL" --format tsv --export $@ @@ -1355,6 +1374,15 @@ ROBOT_RELEASE_IMPORT_MODE_BASE = $(ROBOT) remove --input $< --select imports --t merge $(foreach src, $(OTHER_SRC), --input $(src)){# -#} {% endif %} +MAIN_PRODUCTS = $(sort $(foreach a, $(RELEASE_ARTEFACTS), $(a)) $(ONT)) +MAIN_FILES = $(foreach p, $(MAIN_PRODUCTS), $(foreach f, $(FORMATS), $(p).$(f))){# -#} + {% if project.gzip_main %} \ + $(foreach f, $(FORMATS), $(ONT).$(f).gz){# -#} + {% endif %} + +.PHONY: all_main +all_main: $(MAIN_FILES) + # Release artefacts: main pipelines # ---------------------------------------- @@ -1713,95 +1741,51 @@ $(ONT).owl: $(ONT)-{{ project.primary_release }}.owl {# #} # ---------------------------------------- -# Release assets -# ---------------------------------------- - -MAIN_PRODUCTS = $(sort $(foreach r,$(RELEASE_ARTEFACTS), $(r)) $(ONT)) -MAIN_GZIPPED = {% if project.gzip_main %}$(foreach f,$(FORMATS), $(ONT).$(f).gz){% endif %} -MAIN_FILES = $(foreach n,$(MAIN_PRODUCTS), $(foreach f,$(FORMATS), $(n).$(f))) $(MAIN_GZIPPED) - -.PHONY: all_main -all_main: $(MAIN_FILES) - -# ---------------------------------------- -# Import assets -# ---------------------------------------- - -IMPORT_ROOTS = {% if project.import_group.use_base_merging %} $(IMPORTDIR)/merged_import{% else %}$(patsubst %, $(IMPORTDIR)/%_import, $(IMPORTS)){% endif %}{% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import{% endif %} -IMPORT_OWL_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).owl) -{%- if project.import_group.export_obo %} -IMPORT_OBO_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).obo) -IMPORT_FILES = $(IMPORT_OWL_FILES) $(IMPORT_OBO_FILES) - -{% else %} -IMPORT_FILES = $(IMPORT_OWL_FILES) -{% endif %} - -.PHONY: all_imports -all_imports: $(IMPORT_FILES) - -# ---------------------------------------- -# Subset assets -# ---------------------------------------- - -{% if project.subset_group is defined %} -SUBSETS = {% for x in project.subset_group.products %} {{ x.id }}{% endfor %} -{% else %} -SUBSETS = -{% endif %} -SUBSET_ROOTS = $(patsubst %, $(SUBSETDIR)/%, $(SUBSETS)) -SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(foreach f,$(sort $(FORMATS) tsv), $(n).$(f))) - -.PHONY: all_subsets -all_subsets: $(SUBSET_FILES) - -# ---------------------------------------- -# Mapping assets +# Release management # ---------------------------------------- -{% if project.sssom_mappingset_group is defined %} -MAPPINGS = {% for x in project.sssom_mappingset_group.products %} {{ x.id }}{% endfor %} -{% if project.sssom_mappingset_group.released_products is defined %} -RELEASED_MAPPINGS = {% for x in project.sssom_mappingset_group.released_products %} {{ x.id }}{% endfor %}{% endif %} -{% else %} -MAPPINGS = -{% endif %} -MAPPING_FILES = $(foreach p, $(MAPPINGS), $(MAPPINGDIR)/$(p).sssom.tsv) -RELEASED_MAPPING_FILES = $(foreach p, $(RELEASED_MAPPINGS), $(MAPPINGDIR)/$(p).sssom.tsv) - -.PHONY: all_mappings -all_mappings: $(MAPPING_FILES) - -# ---------------------------------------- -# Bridge assets +# Assets definitions and helper targets # ---------------------------------------- -{% if project.bridge_group is not none -%} -BRIDGES ={% for bridge in project.bridge_group.products %} {{ bridge.id }}{% endfor %} -BRIDGE_FILES = $(foreach bridge, $(BRIDGES), $(BRIDGEDIR)/$(bridge).owl) - -.PHONY: all_bridges -all_bridges: $(BRIDGE_FILES) - -{% else -%} -BRIDGE_FILES = +{% if project.import_group is not none or project.orcidio_support -%} +{# Most assets are defined in the section they belong to, but imports + need to be defined outside of the imports section because we might + have to include the ORCIDIO "import", which is managed separately + from all other imports -#} +IMPORT_ROOTS ={% if project.import_group is not none -%} + {% if project.import_group.use_base_merging -%} $(IMPORTDIR)/merged_import{# -#} + {% else %} $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_import){# -#} + {% endif -%} + {% endif -%} + {% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import{% endif %} +IMPORT_FILES = $(foreach r, $(IMPORT_ROOTS), $(r).owl){# -#} + {% if project.import_group.export_obo %} \ + $(foreach r, $(IMPORT_ROOTS), $(r).obo){# -#} + {% endif %} {% endif -%} - -# ---------------------------------------- -# Release assets -# ---------------------------------------- - -ASSETS = \ - $(IMPORT_FILES) \ - $(MAIN_FILES) \{% if project.use_dosdps %} - $(PATTERN_RELEASE_FILES) \{% endif %}{% if project.use_translations %} - $(TRANSLATION_FILES) \{% endif %} - $(REPORT_FILES) \ - $(SUBSET_FILES) \ - $(MAPPING_FILES) \ - $(BRIDGE_FILES) - +ASSETS = $(MAIN_FILES){# -#} + {% if project.import_group is not none or project.orcidio_support %} \ + $(IMPORT_FILES){# -#} + {% endif -%} + {% if project.use_dosdps %} \ + $(PATTERN_RELEASE_FILES){# -#} + {% endif -%} + {% if project.use_translations %} \ + $(TRANSLATION_FILES){# -#} + {% endif -%} + {% if project.subset_group is not none %} \ + $(SUBSET_FILES){# -#} + {% endif -%} + {% if project.use_mappings %} \ + $(MAPPING_FILES){# -#} + {% endif -%} + {% if project.bridge_group is not none %} \ + $(BRIDGE_FILES){# -#} + {% endif %} \ + $(REPORT_FILES) + +# All the files that are part of a release (excepted mappings). {# This does not include mappings (which can optionally be released), because those are stored in src/mappings, and the fact that this is one directory _above_ the directory where this Makefile is would @@ -1809,104 +1793,103 @@ ASSETS = \ single rsync commmand to copy all release assets except mappings to the release directory). Basically, this is "all release assets that are in this src/ontology directory". -#} -RELEASE_ASSETS = \ - $(MAIN_FILES) {% if project.import_group is defined %}{% if project.import_group.release_imports %}$(IMPORT_FILES) {% endif %}{% endif %}\ - $(SUBSET_FILES){% if project.robot.report.release_reports %} \ - $(REPORT_FILES){% endif %}\ - $(BRIDGE_FILES) +RELEASE_ASSETS = $(MAIN_FILES){# -#} + {% if project.import_group.release_imports %} \ + $(IMPORT_FILES){# -#} + {% endif -%} + {% if project.subset_group is not none %} \ + $(SUBSET_FILES){# -#} + {% endif -%} + {% if project.bridge_group is not none %} \ + $(BRIDGE_FILES){# -#} + {% endif -%} + {% if project.robot.report.release_reports %} \ + $(REPORT_FILES){# -#} + {% endif %} + +# All the files that are part of a release, in their final location +# within the released directory. +{# This must be kept in sync with what the copy_release_files rule below + does -- that is, this should list all files copied by that rule. -#} +RELEASED_ASSETS = $(foreach n, $(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){# -#} + {% if project.sssom_mappingset_group.released_products is defined %} + $(foreach n, $(RELEASED_MAPPINGS), $(RELEASEDIR)/mappings/$(n).sssom.tsv){# -#} + {% endif %} + +# The files that must be removed from the src/ontology directory +# once they have been copied over to the release directory. +CLEANFILES = $(MAIN_FILES) + +{% if project.import_group is not none or project.orcidio_support -%} +.PHONY: all_imports +all_imports: $(IMPORT_FILES) + +{% endif -%} .PHONY: all_assets all_assets: $(ASSETS) {% if project.ensure_valid_rdfxml %}check_rdfxml_assets{% endif %} .PHONY: show_assets show_assets: - echo $(ASSETS) - du -sh $(ASSETS) + @echo $(ASSETS) + @du -sh $(ASSETS) +{# FIXME: This does not really belong here. -#} check_rdfxml_%: % @odk-helper check-rdfxml {% if project.extra_rdfxml_checks %}--jena --rdflib {% endif %}$< .PHONY: check_rdfxml_assets -check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl) +check_rdfxml_assets: $(foreach product, $(MAIN_PRODUCTS), check_rdfxml_$(product).owl) +# Release preparation # ---------------------------------------- -# Release Management -# ---------------------------------------- - -CLEANFILES=$(MAIN_FILES) $(SRCMERGED) $(EDIT_PREPROCESSED) - -# This should be executed by the release manager whenever time comes to make a release. -# It will ensure that all assets/files are fresh, and will copy to release folder +# This is the main release workflow that should be executed by the release +# manager whenever the time comes to make a release. It will ensure that +# all assets are fresh, and copy them over to the release directory. .PHONY: prepare_release prepare_release: all_odk $(MAKE) copy_release_files rm -f $(CLEANFILES) - @echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release \ - on your git hosting site such as GitHub or GitLab" + @echo "INFO: Release files are now in $(RELEASEDIR); now you should commit," + @echo " push and make release on your Git hosting site." .PHONY: prepare_release_fast prepare_release_fast: - $(MAKE) prepare_release IMP=false PAT=false MIR=false COMP=false + $(MAKE_FAST) prepare_release -# This rule does the bulk of the work for prepare_release, copying all -# files to the release directory. It is mostly intended for internal -# use, by the prepare_release rule itself or by other ODK scripts. +{# This rule does the bulk of the work for prepare_release, copying all + files to the release directory. It is mostly intended for internal + use, by the prepare_release rule itself or by other ODK scripts. -#} .PHONY: copy_release_files copy_release_files: - rsync -R $(RELEASE_ASSETS) $(RELEASEDIR){% if project.sssom_mappingset_group is defined %}{% if project.sssom_mappingset_group.released_products is defined %} + rsync -R $(RELEASE_ASSETS) $(RELEASEDIR){# -#} + {% if project.sssom_mappingset_group.released_products is defined %} mkdir -p $(RELEASEDIR)/mappings - cp -rf $(RELEASED_MAPPING_FILES) $(RELEASEDIR)/mappings{% endif %}{% endif %} - -# All released assets, in their final location within the release -{# ODK devs: keep that in sync with the copy_release_files rule above - -- that is, this should list all files copied by that rule. -#} -RELEASE_ASSETS_AFTER_RELEASE=$(foreach n,$(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){% if project.sssom_mappingset_group is defined %}{% if project.sssom_mappingset_group.released_products is defined %} \ - $(foreach n,$(RELEASED_MAPPINGS), $(RELEASEDIR)/mappings/$(n).sssom.tsv){% endif %}{% endif %} + cp -rf $(RELEASED_MAPPING_FILES) $(RELEASEDIR)/mappings{# -#} + {% endif %} .PHONY: show_release_assets show_release_assets: - @echo $(RELEASE_ASSETS_AFTER_RELEASE) - -CURRENT_RELEASE=$(ONTBASE).owl - -$(TMPDIR)/current-release.owl: - wget $(CURRENT_RELEASE) -O $@ - -$(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl - $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ + @echo $(RELEASED_ASSETS) - - - - -# ---------------------------------------- -# Debugging Tools +# Publication # ---------------------------------------- -explain_unsat: $(EDIT_PREPROCESSED) - $(ROBOT) explain -i $< -M unsatisfiability --unsatisfiable random:10 --explanation $(TMPDIR)/$@.md - {% if project.public_release == 'github' -%} -# ---------------------------------------- -# GitHub release -# ---------------------------------------- - GHVERSION = v$(VERSION) .PHONY: public_release public_release: - @echo "Pushing public release $(GHVERSION)..." - @ls -alt $(RELEASE_ASSETS_AFTER_RELEASE) - @gh release create $(GHVERSION) \ - --title "$(VERSION) Release" \ - --generate-notes \ - $(RELEASE_ASSETS_AFTER_RELEASE) + @echo "Pushing public release $(GHVERSION) to GitHub..." + @echo "Assets to be pushed:" + @ls -alt $(RELEASED_ASSETS) + @gh release create $(GHVERSION) + --title "$(VERSION Release" \ + --generate-notes \ + $(RELEASED_ASSETS) {% else -%} -# ---------------------------------------- -# Public release -# ---------------------------------------- .PHONY: public_release public_release: @@ -1916,6 +1899,15 @@ public_release: @false {% endif -%} +{# #} + +# ---------------------------------------- +# Debugging Tools +# ---------------------------------------- + +explain_unsat: $(EDIT_PREPROCESSED) + $(ROBOT) explain -i $< -M unsatisfiability --unsatisfiable random:10 --explanation $(TMPDIR)/$@.md + # ---------------------------------------- # General Validation @@ -1936,6 +1928,14 @@ validate-all-tsv: $(ALL_TSV_FILES) # Editors Utilities # ---------------------------------------- +CURRENT_RELEASE=$(ONTBASE).owl + +$(TMPDIR)/current-release.owl: + wget $(CURRENT_RELEASE) -O $@ + +$(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl + $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ + # This is an experimental target people that want to use ODK Extended Prefix Map (EPM) # can use to pull the (currently inofficial) OBO EPM into the workspace. # Users are instructed to refer to the EPM only through the variable $(EXTENDED_PREFIX_MAP) as From 1b144e7f293b57e566b418fe379b31b9e93c27aa Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 17 Aug 2026 21:49:29 +0100 Subject: [PATCH 15/25] Skip the "export formats" section if not needed. If the only export format is OWL, then we do not need an "export formats" section, since all artefacts are already produced in OWL. Of note, OWL is a _mandatory_ export format: even if the project file does not include it in its `export_formats` setting, it is always forcefully added. Previously, adding OWL back as an export format was done in the Makefile template. Here, we do that at the level of the configuration object in Python code, because this kind of logic does not belong to the template. --- src/incatools/odk/model.py | 4 ++ .../templates/src/ontology/Makefile.jinja2 | 59 ++++++++++--------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/src/incatools/odk/model.py b/src/incatools/odk/model.py index a43371c..fdbbd76 100644 --- a/src/incatools/odk/model.py +++ b/src/incatools/odk/model.py @@ -1209,3 +1209,7 @@ def derive_fields(self) -> None: or self.public_release == "github_python" ): self.public_release = "github" + + # Exporting to OWL is mandatory, even if not explicitly listed + if not "owl" in self.export_formats: + self.export_formats.append("owl") diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index a307a64..8b385c8 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -35,7 +35,7 @@ URIBASE = {{ project.uribase }} ONT = {{ project.id }} ONTBASE = {{ project.uribase }}/{% if project.uribase_suffix is not none %}{{ project.uribase_suffix }}{% else %}{{ project.id }}{% endif %} EDIT_FORMAT = {{ project.edit_format|default('owl') }} -FORMATS = $(sort {% for format in project.export_formats %} {{ format }}{% endfor %} owl) +FORMATS ={% for format in project.export_formats|sort %} {{ format }}{% endfor %} RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}) # Directories @@ -1645,6 +1645,7 @@ $(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) {% endif -%} {% endfor -%} +{% if project.export_formats|length > 1 or project.export_formats[0] != "owl" -%} # Release artefacts: export formats # ---------------------------------------- @@ -1653,12 +1654,12 @@ $(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) variants ($(ONT)-base, $(ONT)-full, etc.), so that we do not have to duplicate all the code below (once for the primary artefact and one when we iterate over the variants. -#} -{% if "obo" in project.export_formats -%} +{% if "obo" in project.export_formats -%} $(ONT).obo: $(ONT).owl $(ROBOT) convert --input $< --check false \ --format obo $(OBO_FORMAT_OPTIONS) --output $@ -{% endif -%} +{% endif -%} {% if "ttl" in project.export_formats -%} {# FIXME: Why are re-annotating? The release.owl product should have been annotated already. @@ -1676,42 +1677,42 @@ $(ONT).json: $(ONT).owl convert --check false --format json --output $@ {% endif -%} -{% for r in project.release_artefacts -%} -{% if r.startswith("custom-") -%} -{% set release = r | replace("custom-","") -%} -{% else -%} -{% set release = "$(ONT)-" ~ r -%} -{% endif -%} -{% if "obo" in project.export_formats -%} +{% for r in project.release_artefacts -%} +{% if r.startswith("custom-") -%} +{% set release = r | replace("custom-","") -%} +{% else -%} +{% set release = "$(ONT)-" ~ r -%} +{% endif -%} +{% if "obo" in project.export_formats -%} {{ release }}.obo: {{ release }}.owl $(ROBOT) convert --input $< --check false \ --format obo $(OBO_FORMAT_OPTIONS) --output $@ -{% endif -%} -{% if "ttl" in project.export_formats -%} +{% endif -%} +{% if "ttl" in project.export_formats -%} {{ release }}.ttl: {{ release }}.owl $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ convert --check false --format ttl --output $@ -{% endif -%} -{% if "json" in project.export_formats -%} +{% endif -%} +{% if "json" in project.export_formats -%} {{ release }}.json: {{ release }}.owl $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ convert --check false --format json --output $@ -{% endif -%} -{% endfor -%} +{% endif -%} +{% endfor -%} -{% if 'db' in project.export_formats -%} +{% if 'db' in project.export_formats -%} -{% if project.use_context -%} +{% if project.use_context -%} CONTEXT_FILE_CSV = $(TMPDIR)/context.csv $(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) @odk-helper context2csv $< > $@ -{% endif -%} +{% endif -%} %.db: %.owl{% if project.use_context %} $(CONTEXT_FILE_CSV){% endif %} @rm -f $*.db $*-relation-graph.tsv.gz .template.db .template.db.tmp @@ -1719,25 +1720,27 @@ $(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) @rm -f $*-relation-graph.tsv.gz .template.db .template.db.tmp @test -f $*.db || (echo "SQLite/SemSQL generation failed" && exit 1) -{% endif -%} +{% endif -%} -{% for format in project.export_formats -%} -{% if project.gzip_main -%} +{% for format in project.export_formats -%} +{% if project.gzip_main -%} $(ONT).{{ format }}.gz: $(ONT).{{ format }} gzip -c $< > $@ -{% endif -%} -{% endfor -%} -{% if 'owl' not in project.export_formats -%} -{% if project.gzip_main -%} +{% endif -%} +{% endfor -%} +{% if 'owl' not in project.export_formats -%} +{% if project.gzip_main -%} $(ONT).owl.gz: $(ONT).owl gzip -c $< > $@ -{% endif -%} +{% endif -%} $(ONT).owl: $(ONT)-{{ project.primary_release }}.owl cp $< $@ -{% endif -%} +{% endif -%} + +{% endif %}{# !if project.export_formats|length > 1 or project.export_formats[0] != "owl" -#} {# #} # ---------------------------------------- From 972ba0b232571929a4e44b65b2ff047c2602b66b Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 17 Aug 2026 22:04:48 +0100 Subject: [PATCH 16/25] Move the definition of import assets to the imports section. Previously (as in: two commits ago), we defined the import assets in the general "assets" section, on the rationale that the definition of those was dependent on both the imports section proper _and_ the ORCIDIO section (if defined). However I now believe this creates needless confusion. Since the ORCIDIO import is not in fact managed by the imports section (it is not really an "import", despite its name; or at the very least it is a special kind of import), it is more consistent _not_ to have it listed in the IMPORT_FILES variable, but to instead explicitly include it in the general ASSETS list. (We still make the `all_imports` target refresh the ORCIDIO import if present though, for convenience.) Two consequences (presumably harmless) of not listing the ORCIDIO import in IMPORT_FILES: * the module will _not_ be exported to OBO even if import_group.export_obo is enabled (this should not be an issue; OBO export for the ORCIDIO module would make no sense anyway, as ORCIDIO only contains individuals and the OBO format cannot represent those); * the module will _not_ be treated as a _release_ asset even if import_group.release_imports is enabled (again, should not be an issue; releasing that module would make little sense -- arguably, releasing _any_ import module makes little sense). --- .../templates/src/ontology/Makefile.jinja2 | 42 ++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 8b385c8..ce214a7 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -504,6 +504,16 @@ $(IMPORTSEED): # Import modules # ---------------------------------------- +IMPORT_ROOTS = {% if project.import_group.use_base_merging -%} + $(IMPORTDIR)/merged_import{# -#} + {% else -%} + $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_import){# -#} + {% endif %} +IMPORT_FILES = $(foreach r, $(IMPORT_ROOTS), $(r).owl){# -#} + {% if project.import_group.export_obo %} \ + $(foreach r, $(IMPORT_ROOTS), $(r).obo){# -#} + {% endif %} + ifeq ($(IMP),true) {% if project.import_group.strip_annotation_properties -%} @@ -704,6 +714,10 @@ $(IMPORTDIR)/%_import.obo: $(IMPORTDIR)/%_import.owl {% endif -%} endif # IMP=true +.PHONY: all_imports +all_imports: $(IMPORT_FILES){# -#} + {% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import.owl{% endif %} + .PHONY: refresh-imports refresh-imports: $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=true clean all_imports @@ -1750,27 +1764,13 @@ $(ONT).owl: $(ONT)-{{ project.primary_release }}.owl # Assets definitions and helper targets # ---------------------------------------- -{% if project.import_group is not none or project.orcidio_support -%} -{# Most assets are defined in the section they belong to, but imports - need to be defined outside of the imports section because we might - have to include the ORCIDIO "import", which is managed separately - from all other imports -#} -IMPORT_ROOTS ={% if project.import_group is not none -%} - {% if project.import_group.use_base_merging -%} $(IMPORTDIR)/merged_import{# -#} - {% else %} $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_import){# -#} - {% endif -%} - {% endif -%} - {% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import{% endif %} -IMPORT_FILES = $(foreach r, $(IMPORT_ROOTS), $(r).owl){# -#} - {% if project.import_group.export_obo %} \ - $(foreach r, $(IMPORT_ROOTS), $(r).obo){# -#} - {% endif %} - -{% endif -%} ASSETS = $(MAIN_FILES){# -#} - {% if project.import_group is not none or project.orcidio_support %} \ + {% if project.import_group is not none %} \ $(IMPORT_FILES){# -#} {% endif -%} + {% if project.orcidio_support %} \ + $(IMPORTDIR)/orcidio_import.owl{# -#} + {% endif -%} {% if project.use_dosdps %} \ $(PATTERN_RELEASE_FILES){# -#} {% endif -%} @@ -1823,12 +1823,6 @@ RELEASED_ASSETS = $(foreach n, $(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){# -#} # once they have been copied over to the release directory. CLEANFILES = $(MAIN_FILES) -{% if project.import_group is not none or project.orcidio_support -%} -.PHONY: all_imports -all_imports: $(IMPORT_FILES) - -{% endif -%} - .PHONY: all_assets all_assets: $(ASSETS) {% if project.ensure_valid_rdfxml %}check_rdfxml_assets{% endif %} From 7de56f1f537e78793483dbe4d4944bf8d047bebf Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 17 Aug 2026 22:23:51 +0100 Subject: [PATCH 17/25] Apply linting rule. --- src/incatools/odk/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/incatools/odk/model.py b/src/incatools/odk/model.py index fdbbd76..f209c3a 100644 --- a/src/incatools/odk/model.py +++ b/src/incatools/odk/model.py @@ -1211,5 +1211,5 @@ def derive_fields(self) -> None: self.public_release = "github" # Exporting to OWL is mandatory, even if not explicitly listed - if not "owl" in self.export_formats: + if "owl" not in self.export_formats: self.export_formats.append("owl") From 6a955ff6006f7fdd78c4a29d5f2626ce9c0ef63d Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 18 Aug 2026 22:10:53 +0100 Subject: [PATCH 18/25] Re-arrange and re-format the last section. --- .../templates/src/ontology/Makefile.jinja2 | 273 +++++++++++------- 1 file changed, 167 insertions(+), 106 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index ce214a7..0175450 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1899,40 +1899,9 @@ public_release: {# #} # ---------------------------------------- -# Debugging Tools +# Utilities # ---------------------------------------- -explain_unsat: $(EDIT_PREPROCESSED) - $(ROBOT) explain -i $< -M unsatisfiability --unsatisfiable random:10 --explanation $(TMPDIR)/$@.md - - -# ---------------------------------------- -# General Validation -# ---------------------------------------- -TSV= -ALL_TSV_FILES={% if project.use_dosdps %}{% if project.pattern_pipelines_group is defined -%}{% for pipeline in project.pattern_pipelines_group.products %}$(DOSDP_TSV_FILES_{{ pipeline.id | upper }}) {% endfor %}{% endif %}$(DOSDP_TSV_FILES_DEFAULT){% endif %} - -validate-tsv: $(TSV) | $(TMPDIR) - for FILE in $< ; do \ - tsvalid $$FILE > $(TMPDIR)/validate.txt; \ - if [ -s $(TMPDIR)/validate.txt ]; then cat $(TMPDIR)/validate.txt && exit 1; fi ; \ - done - -validate-all-tsv: $(ALL_TSV_FILES) - $(MAKE) validate-tsv TSV="$^" - -# ---------------------------------------- -# Editors Utilities -# ---------------------------------------- - -CURRENT_RELEASE=$(ONTBASE).owl - -$(TMPDIR)/current-release.owl: - wget $(CURRENT_RELEASE) -O $@ - -$(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl - $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ - # This is an experimental target people that want to use ODK Extended Prefix Map (EPM) # can use to pull the (currently inofficial) OBO EPM into the workspace. # Users are instructed to refer to the EPM only through the variable $(EXTENDED_PREFIX_MAP) as @@ -1940,113 +1909,202 @@ $(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl $(EXTENDED_PREFIX_MAP): $(ODK_RESOURCES_DIR)/obo.epm.json cp $< $@ -{% if 'obo' in project.edit_format -%} -.PHONY: normalize_obo_src -normalize_obo_src: $(SRC) | all_robot_plugins - $(ROBOT) repair -i $< --merge-axiom-annotations true \ - convert -o $(TMPDIR)/NORM.tmp.obo && \ - mv $(TMPDIR)/NORM.tmp.obo $(SRC) -{%- endif %} - -.PHONY: normalize_src -normalize_src: $(SRC) - $(ROBOT) convert -i $< -f {% if 'obo' == project.edit_format %}obo --check false{% else %}ofn{% endif %} -o $(TMPDIR)/normalise && mv $(TMPDIR)/normalise $< +# Repository maintenance +# ---------------------------------------- .PHONY: validate_idranges validate_idranges: if [ -f {{ project.id }}-idranges.owl ]; then \ - dicer-cli policy --assume-manchester --show-owlapi-error {{ project.id }}-idranges.owl ; \ + dicer-cli policy --assume-manchester --show-owlapi-error {{ project.id }}-idranges.owl ; \ fi -# Deprecated: Use 'sh run.sh odk.py update' without using the Makefile. +# Deprecated: Use 'sh run.sh odk update' without using the Makefile. .PHONY: update_repo update_repo: - odk.py update - -{% if project.documentation is not none %} + odk update + +{% if project.documentation is not none -%} +.PHONY: update_docs: mkdocs gh-deploy --config-file ../../mkdocs.yaml -{%- endif %} -# Note to future generations: computing the real path relative to the -# current directory is a way to ensure we only clean up directories that -# are located below the current directory, regardless of the contents of -# the *DIR variables. +{% endif -%} + +{# Note to future generations: computing the real path relative to the + current directory is a way to ensure we only clean up directories + that are located below the current directory, regardless of the + contents of the *DIR variables. -#} .PHONY: clean -clean:{% if project.use_dosdps %} - $(MAKE) pattern_clean{%- endif %} - for dir in $(MIRRORDIR) $(TMPDIR) ; do \ - reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \ - case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \ +clean: + {% if project.use_dosdps -%} + $(MAKE) pattern_clean + {% endif -%} + for dir in $(MIRRORDIR) $(TMPDIR) ; do \ + reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \ + case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \ done rm -f $(CLEANFILES) - rm -f $(TMPDIR)/download-*.info + +{% if "obo" == project.edit_format -%} +.PHONY: normalize_obo_src +normalize_obo_src: $(SRC) | all_robot_plugins + $(ROBOT) repair --input $< --merge-axiom-annotations true \ + convert --output $< + +{% endif -%} + +.PHONY: normalize_src +normalize_src: $(SRC) + $(ROBOT) convert --input $< --format {# -#} + {% if "obo" == project.edit_format -%} + obo --check false {# -#} + {% else -%} + ofn {# -#} + {% endif -%} + --output $< + +# Debugging tools +# ---------------------------------------- + +.PHONY: explain_unsat +explain_unsat: $(EDIT_PREPROCESSED) + $(ROBOT) explain --input $< \ + --mode unsatisfiability --unsatisfiable random:10 \ + --explanation $(TMPDIR)/$@.md + +# Validation of TSV files +# ---------------------------------------- + +{% if project.use_dosdps -%} +ALL_TSV_FILES = $(DOSDP_TSV_FILES_DEFAULT){# -#} + {% if project.pattern_pipelines_group is not none -%} + {% for pipeline in project.pattern_pipelines_group.products %} \ + $(DOSDP_TSV_FILES_{{ pipeline.id | upper }}){# -#} + {% endfor -%} + {% endif %} + +.PHONY: validate-all-tsv +validate-all-tsv: $(ALL_TSV_FILES) + $(MAKE) validate-tsv TSV="$^" + +{% endif -%} + +{# FIXME: This is of dubious usefulness; if the user needs to validate + an arbitrary TSV file, she might as well run "tsvalid the-file.tsv". -#} +.PHONY: validate-tsv +validate-tsv: $(TSV) | $(TMPDIR) + for FILE in $< ; do \ + tsvalid $$FILE > $(TMPDIR)/validate-tsv.txt ; \ + [ ! -s $(TMPDIR)/validate-tsv.txt ] || (cat $(TMPDIR)/validate-tsv ; false) ; \ + done + +# "Release diff" +# ---------------------------------------- + +# This produces a diff between the last local build of the ontology +# and the version currently available online, +{# FIXME: This is quite OBO-specific. -#} +CURRENT_RELEASE = $(ONTBASE).owl + +$(TMPDIR)/current-release.owl: + wget $(CURRENT_RELEASE) --output-document $@ + +$(REPORTDIR)/release-diff: $(ONT).owl $(TMPDIR)/current-release.owl + $(ROBOT) diff --left $(TMPDIR)/current-release.owl \ + --right $(ONT).owl \ + --labels true --format markdown --output $@ + + +# ---------------------------------------- +# Online help +# ---------------------------------------- .PHONY: help help: - @echo "$$data" + @echo "$$ODK_HELP_MESSAGE" -define data -Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LARGE|PAT)=(false|true)] command +define ODK_HELP_MESSAGE +Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(MIR|IMP|IMP_LARGE|COMP|PAT)=(false|true)] command + + or: odkrun [--lite] [--debug] make [(MIR|IMP|IMP_LARGE|COMP|PAT)=(false|true)] command ---------------------------------------- Command reference ---------------------------------------- Core commands: -* prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports -* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns. -* update_repo: Update the ODK repository setup using the config file {{ project.id }}-odk.yaml (DEPRECATED) -* test: Running all validation tests -* test_fast: Runs the test suite, but without updating imports or components +* prepare_release: Run the entire release pipeline. Use make MIR=false prepare_release to + avoid refreshing remote resources. +* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating + components or recompiling patterns. +* test: Running all validation tests. +* test_fast: Runs the test suite, but without updating imports or components. * odkversion: Show the current version of the ODK Makefile and ROBOT. -* clean: Delete all temporary files -* help: Print ODK Usage information -* public_release: Uploads the release file to a release management system, such as GitHub releases. Must be configured. +* clean: Delete all temporary files. +* help: Print ODK Usage information. +* public_release: Uploads the release file to a release management system, such as GitHub + releases. Must be configured. -{% if project.import_group is defined %} +{% if project.import_group is not none -%} Imports management: * refresh-imports: Refresh all imports and mirrors. * recreate-components: Recreate all components. * no-mirror-refresh-imports: Refresh all imports without downloading mirrors. -* refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled as 'is_large'. -* refresh-%: Refresh a single import, i.e. refresh-go will refresh 'imports/go_import.owl'. -* no-mirror-refresh-%: Refresh a single import without updating the mirror, i.e. refresh-go will refresh 'imports/go_import.owl'. +* refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled + as 'is_large'. +{% if not project.import_group.use_base_merging -%} +* refresh-%: Refresh a single import + (i.e. refresh-go will refresh 'imports/go_import.owl'). +* no-mirror-refresh-%: Refresh a single import without updating the mirror + (i.e. refresh-go will refresh 'imports/go_import.owl'). +{% endif -%} * mirror-%: Refresh a single mirror. -{% endif %}{% if project.use_dosdps %} + +{% endif -%} +{% if project.use_dosdps -%} DOSDP templates -* dosdp: Run the DOSDP patterns pipeline: Run tests, then build OWL files from the tables. -* patterns: Alias of the 'dosdp' command -* pattern_clean: Delete all temporary pattern files -* dosdp_validation: Run all validation checks on DOSDP template files and tables -* pattern_schema_checks: Alias of the 'dosdp_validation' command -* update_patterns: Pull updated patterns listed in dosdp-patterns/external.txt -* dosdp-matches-%: Run the DOSDP matches/query pipeline as configured in your {{ project.id }}-odk.yaml file. +* dosdp: Run the DOSDP patterns pipeline: Run tests, then build OWL files + from the tables. +* patterns: Alias of the 'dosdp' command. +* pattern_clean: Delete all temporary pattern files. +* dosdp_validation: Run all validation checks on DOSDP template files and tables. +* pattern_schema_checks: Alias of the 'dosdp_validation' command. +* update_patterns: Pull updated patterns listed in dosdp-patterns/external.txt. +* dosdp-matches-%: Run the DOSDP matches/query pipeline as configured in your project. * dosdp-docs-%: Generate the documentation for a given DOSDP pipeline. -{% endif %} + +{% endif -%} Editor utilities: -* validate_idranges: Make sure your ID ranges file is formatted correctly -* normalize_src: Load and save your {{ project.id }}-edit file after you to make sure its serialised correctly{% if 'obo' in project.edit_format %} -* normalize_obo_src: Load and save your {{ project.id }}-edit.obo file after you to merge duplicate annotation assertions{% endif %} -* explain_unsat: If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes -* validate-all-tsv: Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files -* validate-tsv: Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv. -* release_diff: Create a diff between the current release and the new release - -Additional build commands (advanced users) -* all: Run the entire pipeline (like prepare_release), but without copying the release files to the release directory. -* all_subsets: Build all subsets -* custom_reports: Generate all custom sparql reports you have configured in your {{ project.id }}-odk.yaml file. -* all_assets: Build all assets -* show_assets: Print a list of all assets that would be build by the release pipeline -* all_mappings: Update all SSSOM mapping sets - -Additional QC commands (advanced users) -* robot_reports: Run all configured ROBOT reports -* validate_profile_%: Run an OWL2 DL profile validation check, for example validate_profile_{{ project.id }}-edit.owl. -* reason_test: Run a basic reasoning test - -Examples: +* validate_idranges: Make sure your ID ranges file is formatted correctly. +* normalize_src: Load and save your {{ project.id }}-edit file after you to make sure it is + serialised correctly +{% if "obo" == project.edit_format -%} +* normalize_obo_src: Load and save your {{ project.id }}-edit.obo file after you to merge + duplicate annotation assertions. +{% endif -%} +* explain_unsat: Produce explanations (in $(TMPDIR)/explain_unsat.md) for a random sample of + unsatisfiable classes in your ontology (if any). +{% if project.use_dosdps -%} +* validate-all-tsv: Check all your tsv files for possible problems in syntax. +{% endif -%} +* release_diff: Create a diff between the current release and the new release + +Additional build commands (advanced users): +* all: Run the entire pipeline (like prepare_release), but without copying the + release files to the release directory. +* all_subsets: Build all subsets. +* custom_reports: Generate all custom sparql reports you have configured in your project. +* all_assets: Build all assets. +* show_assets: Print a list of all assets that would be build by the release pipeline. +* all_mappings: Update all SSSOM mapping sets. + +Additional QC commands (advanced users): +* robot_reports: Run all configured ROBOT reports. +* validate_profile_%: Run an OWL2 DL profile validation check on an arbitrary file + (i.e. validate_profile_{{ project.id }}-edit.owl). +* reason_test: Run a basic reasoning test. + +Examples: * sh run.sh make IMP=false prepare_release * sh run.sh make test @@ -2055,13 +2113,16 @@ Tricks: invoke the 'clean' target (which will wipe out intermediate files) or touch a file that your target depends on (typically the -edit file). * Use the IMAGE parameter to the run.sh script to use a different image like odklite + (or use odkrun's --image option). * Use ODK_DEBUG=yes sh run.sh make ... to print information about timing and debugging + (or use odkrun's --debug option). Updating the repository: (to apply changes to the ODK configuration or switch to a newer ODK version) * sh run.sh update_repo + (or odkrun odk update) endef -export data +export ODK_HELP_MESSAGE include {{ project.id }}.Makefile From 33bbcba79b8156bf2b1e091edc26adcd904b1b4d Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Wed, 19 Aug 2026 00:34:17 +0100 Subject: [PATCH 19/25] Move the rdfxml test to the main artefacts section. The test for RDF/XML correctness is only about the main products, so it belongs to the section that creates those products. --- .../templates/src/ontology/Makefile.jinja2 | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 0175450..21c9b34 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1395,8 +1395,17 @@ MAIN_FILES = $(foreach p, $(MAIN_PRODUCTS), $(foreach f, $(FORMATS), $(p).$(f {% endif %} .PHONY: all_main -all_main: $(MAIN_FILES) +all_main: $(MAIN_FILES){# -#} + {% if project.ensure_valid_rdfxml %} check_rdfxml_products{% endif %} +{% if project.ensure_valid_rdfxml -%} +.PHONY: check_rdfxml_products +check_rdfxml_products: $(foreach product, $(MAIN_PRODUCTS), check_rdfxml_$(product).owl) + +check_rdfxml_%: % + @odk-helper check-rdfxml {% if project.extra_rdfxml_checks %}--jena --rdflib {% endif %}$< + +{% endif -%} # Release artefacts: main pipelines # ---------------------------------------- @@ -1824,20 +1833,13 @@ RELEASED_ASSETS = $(foreach n, $(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){# -#} CLEANFILES = $(MAIN_FILES) .PHONY: all_assets -all_assets: $(ASSETS) {% if project.ensure_valid_rdfxml %}check_rdfxml_assets{% endif %} +all_assets: $(ASSETS) .PHONY: show_assets show_assets: @echo $(ASSETS) @du -sh $(ASSETS) -{# FIXME: This does not really belong here. -#} -check_rdfxml_%: % - @odk-helper check-rdfxml {% if project.extra_rdfxml_checks %}--jena --rdflib {% endif %}$< - -.PHONY: check_rdfxml_assets -check_rdfxml_assets: $(foreach product, $(MAIN_PRODUCTS), check_rdfxml_$(product).owl) - # Release preparation # ---------------------------------------- From 2f324abc24b1bfd4d694466f4ce21814429e4126 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Wed, 19 Aug 2026 23:49:36 +0100 Subject: [PATCH 20/25] Make another formatting pass. Make another pass (hopefully the last one) to make the formatting more uniform. Notably, make sure to avoid using a mix of tabs and spaces for indenting. Only use tabs when they are required by Make's syntax (first character of every line within a rule), otherwise use spaces. Also always favour "post-block" space control over "pre-block" whenever possible, unless the use of post-block control happens to be more cumbersome than pre-block (this is sometimes the case). --- .../templates/src/ontology/Makefile.jinja2 | 403 +++++++++++------- 1 file changed, 247 insertions(+), 156 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 21c9b34..bf9b9c9 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -12,13 +12,14 @@ # Generated using the Ontology Development Kit (ODK) # ODK Version: {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %} # ---------------------------------------- -# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use {{ project.id }}.Makefile instead +# IMPORTANT: DO NOT EDIT THIS FILE. +# To override default make goals, edit {{ project.id }}.Makefile instead. {{ project.custom_makefile_header }} {% if project.config_hash -%} # Fingerprint of the configuration file when this Makefile was last generated -CONFIG_HASH= {{ project.config_hash }} +CONFIG_HASH = {{ project.config_hash }} {% endif -%} # ---------------------------------------- @@ -33,10 +34,20 @@ CONFIG_HASH= {{ project.config_hash }} OBOBASE = http://purl.obolibrary.org/obo URIBASE = {{ project.uribase }} ONT = {{ project.id }} -ONTBASE = {{ project.uribase }}/{% if project.uribase_suffix is not none %}{{ project.uribase_suffix }}{% else %}{{ project.id }}{% endif %} +ONTBASE = {{ project.uribase }}/{% if project.uribase_suffix is not none -%} + {{ project.uribase_suffix -}} + {% else -%} + {{ project.id -}} + {% endif %} EDIT_FORMAT = {{ project.edit_format|default('owl') }} FORMATS ={% for format in project.export_formats|sort %} {{ format }}{% endfor %} -RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}) +RELEASE_ARTEFACTS ={% for release in project.release_artefacts %} {# -#} + {% if release.startswith("custom") -%} + {{ release | replace("custom-", "") -}} + {% else -%} + $(ONT)-{{ release -}} + {% endif -%} + {% endfor %} # Directories RELEASEDIR = ../.. @@ -66,7 +77,16 @@ ODK_RESOURCES_DIR ?= /tools/resources # Files SRC = $(ONT)-edit.$(EDIT_FORMAT) -OTHER_SRC = {% if project.use_dosdps %}$(PATTERNDIR)/definitions.owl {% endif %}{% if project.components is defined %}{% for component in project.components.products %}$(COMPONENTSDIR)/{{ component.filename }} {% endfor %}{% endif %} +OTHER_SRC ={% if project.use_dosdps %} {# -#} + $(PATTERNDIR)/definitions.owl{# -#} + {% endif -%} + {% if project.components is defined -%} + {% for component in project.components.products -%} + {% if not loop.first or project.use_dosdps %} \ + {% endif %} {# -#} + $(COMPONENTSDIR)/{{ component.filename -}} + {% endfor -%} + {% endif %} CATALOG = {{ project.catalog_file }} {% if project.use_context -%} CONTEXT_FILE = config/context.json @@ -99,10 +119,10 @@ VERSION = $(TODAY) # Commands MAKE_FAST = $(MAKE) IMP=false PAT=false COMP=false MIR=false ANNOTATE_ONTOLOGY_VERSION = annotate --version-iri $(ONTBASE)/releases/$(VERSION)/$@ \ - --annotation owl:versionInfo $(VERSION) + --annotation owl:versionInfo $(VERSION) ANNOTATE_CONVERT_FILE = $(ANNOTATE_ONTOLOGY_VERSION) --ontology-iri $(ONTBASE)/$@ \ - convert --format {{ project.import_component_format|default('ofn') }} \ - --output $@ + convert --format {{ project.import_component_format|default('ofn') }} \ + --output $@ # Debugging ifeq ($(ODK_DEBUG),yes) @@ -145,28 +165,35 @@ PAT = true all: all_odk .PHONY: all_odk -all_odk: odkversion{% if project.config_hash %} config_check{% endif %} test custom_reports all_assets{% if project.release_diff %} release_diff{% endif %} +all_odk: odkversion {# -#} + {% if project.config_hash %}config_check {% endif -%} + test custom_reports all_assets{# -#} + {% if project.release_diff %} release_diff{% endif %} .PHONY: odkversion odkversion: - @echo "ODK Makefile {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %}" + @echo "INFO: ODK Makefile {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %}" @odk-helper info --tools {% if project.config_hash -%} .PHONY: config_check config_check: @if [ "$$(tr -d '\r' < $(ONT)-odk.yaml | sha256sum | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \ - echo "Repository is up-to-date." ; else \ - echo "Your ODK configuration has changed since this Makefile was generated." ; \ - echo "You may need to run 'sh run.sh update_repo'." ; \ + echo "INFO: Repository is up-to-date." ; else \ + echo "WARNING: Your ODK configuration has changed since this Makefile was generated." ; \ + echo " You may need to run 'sh run.sh update_repo'." ; \ fi {% endif -%} .PHONY: test -test: odkversion validate_idranges{% if project.use_dosdps %} dosdp_validation{% endif %} reason_test sparql_test robot_reports{% if project.robot.report.ensure_owl2dl_profile|default(true) %} \ - $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{% endif %} - @echo "Finished running all tests successfully." +test: odkversion validate_idranges {# -#} + {% if project.use_dosdps %}dosdp_validation {% endif -%} + reason_test sparql_test robot_reports{# -#} + {% if project.robot.report.ensure_owl2dl_profile|default(true) %} \ + $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{# -#} + {% endif %} + @echo "INFO: Finished running all tests successfully." .PHONY: test_fast test_fast: @@ -175,15 +202,16 @@ test_fast: .PHONY: reason_test reason_test: $(EDIT_PREPROCESSED) $(ROBOT) reason --input $< \ - --reasoner $(REASONER) \ - --equivalent-classes-allowed {{ project.allow_equivalents }} \ - --exclude-tautologies {{ project.exclude_tautologies }} + --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} .PHONY: release_diff release_diff: $(REPORTDIR)/release-diff.md -$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR) -{%- if project.bridge_group is not none %} $(BRIDGEDIR){% endif %}: +$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR){# -#} +{% if project.bridge_group is not none %} $(BRIDGEDIR){# -#} +{% endif %}: mkdir -p $@ @@ -207,33 +235,34 @@ extra_robot_plugins: {% for plugin in project.robot.plugins %} $(ROBOT_PLUGINS_D # Install all ROBOT plugins to the runtime plugins directory .PHONY: all_robot_plugins all_robot_plugins: $(foreach plugin,$(notdir $(wildcard $(ODK_RESOURCES_DIR)/robot/plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \ - $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \ - custom_robot_plugins {% if project.robot.plugins is not none %}extra_robot_plugins {% endif %} \ + $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \ + custom_robot_plugins {% if project.robot.plugins is not none %}extra_robot_plugins {% endif %} \ # Default rule to install plugins $(ROBOT_PLUGINS_DIRECTORY)/%.jar: @mkdir -p $(ROBOT_PLUGINS_DIRECTORY) - @if [ -f ../../plugins/$*.jar ]; then \ - ln ../../plugins/$*.jar $@ ; \ + @if [ -f ../../plugins/$*.jar ]; then \ + ln ../../plugins/$*.jar $@ ; \ elif [ -f $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar ]; then \ - cp $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar $@ ; \ + cp $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar $@ ; \ fi {% if project.robot.plugins is not none -%} # Additional rules for plugins defined in the ODK configuration {% for plugin in project.robot.plugins -%} $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar: -{% if plugin.mirror_from -%} -{# #} curl -L -o $@ {{ plugin.mirror_from }} -{% else -%} -{# #} @echo "ERROR: No URL has been provided for this plugin; you must override" + {# -#} + {% if plugin.mirror_from -%} + curl --location --output $@ {{ plugin.mirror_from -}} + {% else -%} + @echo "ERROR: No URL has been provided for this plugin; you must override" @echo " this rule in {{ project.id }}.Makefile to install it." - @false -{% endif -%} -{# #} -{% endfor -%} + @false{# -#} + {% endif %} -{% endif %} +{% endfor -%} +{% endif -%} +{# #} # ---------------------------------------- # Internal intermediate products # ---------------------------------------- @@ -266,8 +295,10 @@ OBO_REPORT ={% for x in project.robot.report.report_on|default(["ed ALIGNMENT_REPORT ={% for x in project.robot.report.report_on|default(["edit"]) -%} {% if x=="edit" %} $(SRC){% else %} {{ x }}{% endif %}-align-report{% endfor %} {% endif -%} -REPORTS = $(OBO_REPORT){% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} - {# #} $(ALIGNMENT_REPORT){% endif %} +REPORTS = $(OBO_REPORT){# -#} + {% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology %} {# -#} + $(ALIGNMENT_REPORT){# -#} + {% endif %} REPORT_FILES = $(foreach report, $(REPORTS), $(REPORTDIR)/$(report).tsv) REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}--labels true{% endif %} @@ -281,30 +312,30 @@ all_reports: custom_reports robot_reports $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) $(ROBOT) report --input $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) \ - --fail-on $(REPORT_FAIL_ON) \ - {% if project.robot.report.use_base_iris -%} - {% if project.namespaces is not none -%} - {% for iri in project.namespaces %}--base-iri {{ iri }} \ - {% endfor -%} - {% else -%}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ - --base-iri $(URIBASE)/{{ project.id }} \ - {% endif -%} - {% endif -%} - --print 5 --output $@ + --fail-on $(REPORT_FAIL_ON) \ + {% if project.robot.report.use_base_iris -%} + {% if project.namespaces is not none -%} + {% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor -%} + {% else -%}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ + --base-iri $(URIBASE)/{{ project.id }} \ + {% endif -%} + {% endif -%} + --print 5 --output $@ {% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} $(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins $(ROBOT) odk:check-align --input $< --reasoner $(REASONER) \ - --upper-ontology-iri {{ project.robot.report.upper_ontology }} \ - {% if project.robot.report.use_base_iris %}--detail BASE-ROOT \ - {% if project.namespaces is not none -%} - {% for iri in project.namespaces %}--base-iri {{ iri }} \ - {% endfor -%} - {% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ - --base-iri $(URIBASE)/{{ project.id }} \ - {% endif -%} - {% endif -%} - --report-output $@ + --upper-ontology-iri {{ project.robot.report.upper_ontology }} \ + {% if project.robot.report.use_base_iris %}--detail BASE-ROOT \ + {% if project.namespaces is not none -%} + {% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor -%} + {% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ + --base-iri $(URIBASE)/{{ project.id }} \ + {% endif -%} + {% endif -%} + --report-output $@ {% endif -%} @@ -320,12 +351,12 @@ check_for_robot_updates: # The merge step is necessary to avoid undeclared entity violations. $(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) $(ROBOT) merge --input $< \ - validate-profile --profile DL \ - --output $@ + validate-profile --profile DL \ + --output $@ .PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt - @echo "$* profile validation completed." + @echo "INFO: $* profile validation completed." # SPARQL-based checks and reports # ---------------------------------------- @@ -337,7 +368,11 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt {# FIXME: The default list should be computed in the seeding script, not here. -#} SPARQL_VALIDATION_CHECKS ={% for x in project.robot.report.custom_sparql_checks|default(["owldef-self-reference", "iri-range", "label-with-iri", "multiple-replaced_by"]) %} {{ x }}{% endfor %} -sparql_test:{% for x in project.robot.report.sparql_test_on|default(["edit"]) %} {% if x =="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %}{% endfor %} | $(REPORTDIR) +sparql_test:{% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} + {% if x =="edit" %} $(SRCMERGED){# -#} + {% else %} {{ x -}} + {% endif -%} + {% endfor %} | $(REPORTDIR) ifneq ($(SPARQL_VALIDATION_CHECKS),) {% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} {# #} $(ROBOT) verify --input {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %} \ @@ -386,13 +421,13 @@ download-mirror-{{ ont.id }}: | $(TMPDIR) odk-helper download --output $(TMPDIR)/$@.owl \ --reference $(MIRRORDIR)/{{ ont.id }}.owl \ --max-retry {{ project.import_group.mirror_retry_download }} \ - {% if ont.mirror_from -%} - {{ ont.mirror_from }} - {%- elif ont.use_variant is not none -%} - $(OBOBASE)/{{ ont.id }}/{{ ont.id }}-{{ ont.use_variant }}.owl{% if ont.use_gzipped %}.gz{% endif %} - {%- else -%} - $(OBOBASE)/{{ ont.id }}.owl{% if ont.use_gzipped %}.gz{% endif %} - {%- endif %} + {% if ont.mirror_from -%} + {{ ont.mirror_from -}} + {% elif ont.use_variant is not none -%} + $(OBOBASE)/{{ ont.id }}/{{ ont.id }}-{{ ont.use_variant }}.owl{% if ont.use_gzipped %}.gz{% endif -%} + {% else -%} + $(OBOBASE)/{{ ont.id }}.owl{% if ont.use_gzipped %}.gz{% endif -%} + {% endif %} {% if ont.mirror_type == 'custom' -%} $(MIRRORDIR)/{{ ont.id }}.owl: @@ -522,19 +557,24 @@ ANNOTATION_PROPERTIES ={% for p in project.import_group.annotation_properties %} {% if project.import_group.use_base_merging -%} ALL_TERMS = $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_terms.txt) -{% if 'slme' == project.import_group.module_type -%} +{% if "slme" == project.import_group.module_type -%} $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ $(IMPORTSEED) | all_robot_plugins - $(ROBOT) merge --input $<{% if project.import_group.exclude_iri_patterns is not none -%} - {%- for pattern in project.import_group.exclude_iri_patterns %} \ - remove --select "{{ pattern }}"{% endfor %}{% endif %} \ + $(ROBOT) merge --input $<{# -#} + {% if project.import_group.exclude_iri_patterns is not none -%} + {% for pattern in project.import_group.exclude_iri_patterns %} \ + remove --select "{{ pattern }}"{# -#} + {% endfor -%} + {% endif %} \ extract $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ --force true --copy-ontology-annotations false \ - --individuals {{ project.import_group.slme_individuals }} \ - --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} + --individuals {{ project.import_group.slme_individuals }} {# -#} + --method {{ project.import_group.module_type_slme }} \ + {% if project.import_group.strip_annotation_properties -%} remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} + --select complement --select annotation-properties \ + {% endif -%} odk:normalize --base-iri {{ project.uribase }} \ --subset-decls true --synonym-decls true \ repair --merge-axiom-annotations true \ @@ -551,24 +591,26 @@ $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ {% else %}{# !project.import_group.use_base_merging -#} ## Default module type ({{ project.import_group.module_type }}) -{% if 'slme' == project.import_group.module_type -%} +{% if "slme" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ odk:normalize --add-source true \ extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ --force true --copy-ontology-annotations true \ - --individuals {{ project.import_group.slme_individuals }} \ - --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} + --individuals {{ project.import_group.slme_individuals }} {# -#} + --method {{ project.import_group.module_type_slme }} \ + {% if project.import_group.strip_annotation_properties -%} remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} + --select complement --select annotation-properties \ + {% endif -%} odk:normalize --base-iri {{ project.uribase }} \ --subset-decls true --synonym-decls true \ repair --merge-axiom-annotations true \ $(ANNOTATE_CONVERT_FILE) -{% elif 'minimal' == project.import_group.module_type -%} +{% elif "minimal" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ @@ -588,7 +630,7 @@ $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ --select "classes individuals annotation-properties" \ $(ANNOTATE_CONVERT_FILE) -{% elif 'mirror' == project.import_group.module_type -%} +{% elif "mirror" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ odk:normalize --base-iri {{ project.uribase }} \ @@ -597,7 +639,7 @@ $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl | all_robot_plugins repair --merge-axiom-annotations true \ $(ANNOTATE_CONVERT_FILE) -{% elif 'filter' == project.import_group.module_type -%} +{% elif "filter" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ $(IMPORTSEED) | all_robot_plugins $(ROBOT) merge --input $< \ @@ -614,7 +656,7 @@ $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ repair --merge-axiom-annotations true \ $(ANNOTATE_CONVERT_FILE) -{% elif 'custom' == project.import_group.module_type -%} +{% elif "custom" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl @echo "ERROR: You have configured the default module type to be custom;" @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" @@ -636,39 +678,47 @@ $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl {% if ont.is_large -%} ifeq ($(IMP_LARGE),true) {% endif -%} -{% if 'slme' == ont.module_type -%} +{% if "slme" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ odk:normalize --add-source true \ extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ --copy-ontology-annotations true --force true \ --individuals {{ ont.slme_individuals }} \ - --method {{ ont.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} + --method {{ ont.module_type_slme }} \ + {% if project.import_group.strip_annotation_properties -%} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + {% for p in ont.annotation_properties -%} + --term {{ p }} \ + {% endfor -%} --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} + --select complement --select annotation-properties \ + {% endif -%} odk:normalize --base-iri {{ project.uribase }} \ --subset-decls true --synonym-decls true \ repair --merge-axiom-annotations true \ $(ANNOTATE_CONVERT_FILE) -{% elif 'filter' == ont.module_type -%} +{% elif "filter" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ odk:normalize --add-source true \ extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ --copy-ontology-annotations true --force true --method BOT \ - remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} + remove --axioms external --preserve-structure false --trim false \ + {%- for iri in ont.base_iris %} + --base-iri {{ iri }} \ + {%- endfor %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + {%- for p in ont.annotation_properties %} + --term {{ p }} \ + {%- endfor %} --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ --select complement \ odk:normalize --base-iri {{ project.uribase }} \ --subset-decls true --synonym-decls true \ repair --merge-axiom-annotations true \ $(ANNOTATE_CONVERT_FILE) -{% elif 'mirror' == ont.module_type -%} +{% elif "mirror" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ odk:normalize --base-iri {{ project.uribase }} \ @@ -676,25 +726,29 @@ $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl | all_robot_ --add-source true \ repair --merge-axiom-annotations true \ $(ANNOTATE_CONVERT_FILE) -{% elif 'minimal' == ont.module_type -%} +{% elif "minimal" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ odk:normalize --add-source true \ extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ --copy-ontology-annotations true --force true --method BOT \ - remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %} + remove --axioms external --preserve-structure false --trim false \ + {%- for iri in ont.base_iris %} + --base-iri {{ iri }} \ + {%- endfor %} odk:normalize --base-iri {{ project.uribase }} --subset-decls true \ --synonym-decls true \ repair --merge-axiom-annotations true \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + {%- for p in ont.annotation_properties %} + --term {{ p }} \ + {%- endfor %} --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ --select complement \ --select "classes individuals annotation-properties" \ $(ANNOTATE_CONVERT_FILE) {% elif 'custom' -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if 'no_mirror' != ont.mirror_type %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if "no_mirror" != ont.mirror_type %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} @echo "ERROR: You have configured {{ ont.id }} as a custom module;" @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" @false @@ -774,31 +828,44 @@ download-component-{{ component.filename }}: | $(TMPDIR) $(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }} download-component-{{ component.filename }} @if [ -f $(TMPDIR)/download-component-{{ component.filename }} ]; then \ - $(ROBOT) merge -i $(TMPDIR)/download-component-{{ component.filename }} \{% if component.make_base %} + $(ROBOT) merge --input $(TMPDIR)/download-component-{{ component.filename }} \ + {% if component.make_base -%} remove {% for iri in component.base_iris %}--base-iri {{ iri }} \ - {% endfor %}--axioms external --preserve-structure false --trim false \{% endif %} - $(ANNOTATE_CONVERT_FILE) ; \ + {% endfor -%} + --axioms external --preserve-structure false --trim false \ + {% endif -%} + $(ANNOTATE_CONVERT_FILE) ; \ fi .PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} endif # MIR=true {% elif component.use_template -%} -$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }}{% for template in component.templates %} \ - $(TEMPLATEDIR)/{{ template }}{% endfor %} - $(ROBOT) template {% if component.template_options is not none %}{{ component.template_options }}{% endif %} \ - {% for template in component.templates %}--template $(TEMPLATEDIR)/{{ template }} \ - {% endfor %}$(ANNOTATE_CONVERT_FILE) +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename -}} + {% for template in component.templates %} \ + $(TEMPLATEDIR)/{{ template -}} + {% endfor %} + $(ROBOT) template {% if component.template_options is not none -%} + {{ component.template_options }} \ + {% endif -%} + {% for template in component.templates -%} + --template $(TEMPLATEDIR)/{{ template }} \ + {% endfor -%} + $(ANNOTATE_CONVERT_FILE) .PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} {% elif component.use_mappings -%} -$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }}{% for mapping in component.mappings %} \ - $(MAPPINGDIR)/{{ mapping }}{% endfor %} \ - | all_robot_plugins +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename -}} + {% for mapping in component.mappings %} \ + $(MAPPINGDIR)/{{ mapping -}} + {% endfor %} \ + | all_robot_plugins $(ROBOT) --add-prefix 'sssom: https://w3id.org/sssom/' \ - --add-prefix 'semapv: http://w3id.org/semapv/vocab/' \ - sssom:inject {% for mapping in component.mappings %}--sssom $(MAPPINGDIR)/{{ mapping }} \ - {% endfor %}--create --direct \ + --add-prefix 'semapv: https://w3id.org/semapv/vocab/' \ + sssom:inject {% for mapping in component.mappings -%} + --sssom $(MAPPINGDIR)/{{ mapping }} \ + {% endfor -%} + --create --direct \ $(ANNOTATE_CONVERT_FILE) .PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} @@ -811,13 +878,17 @@ all_components: $(OTHER_SRC) .PHONY: recreate-components recreate-components: - $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ - {% endfor %}COMP=true IMP=false MIR=true PAT=true all_components + $(MAKE) {% for component in project.components.products -%} + --assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ + {% endfor -%} + COMP=true IMP=false MIR=true PAT=true all_components .PHONY: no-mirror-recreate-components no-mirror-recreate-components: - $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ - {% endfor %}COMP=true IMP=false MIR=false PAT=true all_components + $(MAKE) {% for component in project.components.products -%} + --assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ + {% endfor -%} + COMP=true IMP=false MIR=false PAT=true all_components .PHONY: recreate-% recreate-%: @@ -860,9 +931,11 @@ ifeq ($(MIR),true) .PHONY: download-mirror-orcidio download-mirror-orcidio: | $(TMPDIR) $(MIRRORDIR) @odk-helper download --output $(TMPDIR)/$@.owl \ - --reference $(MIRRORDIR)/orcidio.owl{% if project.import_group is defined %} \ - --max-retry {{ project.import_group.mirror_retry_download }}{% endif %} \ - https://w3id.org/orcidio/orcidio.owl + --reference $(MIRRORDIR)/orcidio.owl \ + {% if project.import_group is defined -%} + --max-retry {{ project.import_group.mirror_retry_download }} \ + {% endif -%} + https://w3id.org/orcidio/orcidio.owl $(MIRRORDIR)/orcidio.owl: download-mirror-orcidio @if [ -f $(TMPDIR)/download-mirror-orcidio.owl ]; then \ @@ -916,9 +989,11 @@ $(TMPDIR)/{{ mapping.id }}.obographs.json: {{ mapping.source_file }} convert --check false --format json --output $@ $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: $(TMPDIR)/{{ mapping.id }}.obographs.json - $(SSSOMPY) parse $< --input-format obographs-json {% if mapping.sssom_tool_options %}\ - {{ mapping.sssom_tool_options }} \ - {% else %}{% endif %}--output $@ + $(SSSOMPY) parse $< --input-format obographs-json {# -#} + {% if mapping.sssom_tool_options %}\ + {{ mapping.sssom_tool_options }} \ + {% endif -%} + --output $@ {% elif project.sssom_mappingset_group.mapping_extractor == "sssom-java" -%} $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: {{ mapping.source_file }} | all_robot_plugins @@ -935,7 +1010,8 @@ $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: {% elif mapping.maintenance == "merged" -%} $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% for source in mapping.source_mappings %} $(MAPPINGDIR)/{{ source }}.sssom.tsv {%- if not loop.last %} \ - {% endif %}{% endfor %} + {% endif -%} + {% endfor %} sssom-cli --output $@ $^ {% elif mapping.maintenance == "mirror" -%} @@ -976,16 +1052,20 @@ all_bridges: $(BRIDGE_FILES) {% for bridge in project.bridge_group.products -%} {% if bridge.bridge_type == "sssom" -%} -$(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt{% for source in bridge.sources %} \ - $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} \ - | $(BRIDGEDIR) all_robot_plugins - $(ROBOT) sssom:inject {% for source in bridge.sources %}--sssom $(MAPPINGDIR)/{{ source }}.sssom.tsv \ - {% endfor %}--ruleset $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ +$(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ + {% for source in bridge.sources -%} + $(MAPPINGDIR)/{{ source }}.sssom.tsv \ + {% endfor -%} + | $(BRIDGEDIR) all_robot_plugins + $(ROBOT) sssom:inject {% for source in bridge.sources -%} + --sssom $(MAPPINGDIR)/{{ source }}.sssom.tsv \ + {% endfor -%} + --ruleset $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ --create \ annotate --ontology-iri $(ONTBASE)/bridges/{{ bridge.id }}.owl \ --version-iri $(ONTBASE)/releases/$(VERSION)/bridges/{{ bridge.id }}.owl \ --annotation owl:versionInfo $(VERSION) \ - convert -f ofn -o $@ + convert --format ofn --output $@ {% elif bridge.bridge_type == "custom" -%} $(BRIDGEDIR)/{{ bridge.id }}.owl: @@ -1013,10 +1093,11 @@ ALL_PATTERN_NAMES = $(strip $(basename $(notdir $(ALL_PATTERN_FILES)))) PATTERN_CLEAN_FILES = $(PATTERNDIR)/all_pattern_terms.txt \ $(DOSDP_OWL_FILES_DEFAULT) $(DOSDP_TERM_FILES_DEFAULT) - {%- if project.pattern_pipelines_group is defined -%} - {% for pipeline in project.pattern_pipelines_group.products %} \ + {%- if project.pattern_pipelines_group is defined %} + {%- for pipeline in project.pattern_pipelines_group.products %} \ $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}) - {%- endfor %}{% endif %} + {%- endfor %} + {%- endif %} EXTERNAL_PATTERN_URLS = $(strip $(file <$(PATTERNDIR)/dosdp-patterns/external.txt)) EXTERNAL_PATTERN_FILES = $(notdir $(EXTERNAL_PATTERN_URLS)) @@ -1073,8 +1154,10 @@ $(DOSDP_OWL_FILES_DEFAULT): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(AL --infile=$(PATTERNDIR)/data/default/ \ --template=$(PATTERNDIR)/dosdp-patterns \ --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ - --ontology=$< {% if project.dosdp_tools_options %}\ - {{ project.dosdp_tools_options }} {% endif %}\ + --ontology=$< \ + {% if project.dosdp_tools_options -%} + {{ project.dosdp_tools_options }} \ + {% endif -%} --outfile=$(PATTERNDIR)/data/default ; \ fi @@ -1087,9 +1170,15 @@ dosdp-docs-default: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(DOSDP_YAML --infile=$(PATTERNDIR)/data/default \ --template=$(PATTERNDIR)/dosdp-patterns \ --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ - --outfile=$(DOCSDIR)/patterns/default{% if project.repo_url %} \ - --data-location-prefix={{ project.repo_url }}/src/patterns/data/default{% elif project.github_org and project.repo %} \ - --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/default{% endif %} + {% if project.repo_url -%} + --data-location-prefix={{ project.repo_url }}/src/patterns/data/default \ + {% elif project.github_org and project.repo -%} + --data-location-prefix=https://github.com/{{ project.github_org -}} + /{{ project.repo -}} + /tree/{{ project.git_main_branch -}} + /src/patterns/data/default \ + {% endif -%} + --outfile=$(DOCSDIR)/patterns/default {% if project.pattern_pipelines_group is defined -%} {% for pipeline in project.pattern_pipelines_group.products -%} @@ -1108,8 +1197,10 @@ $(DOSDP_OWL_FILES_{{ PID }}): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ PID }}) --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ --template=$(PATTERNDIR)/dosdp-patterns/ \ --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ PID }})" \ - --ontology=$< {% if pipeline.dosdp_tools_options %}\ - {{ pipeline.dosdp_tools_options }} {% endif %}\ + --ontology=$< \ + {% if pipeline.dosdp_tools_options -%} + {{ pipeline.dosdp_tools_options }} \ + {% endif -%} --outfile=$(PATTERNDIR)/data/{{ pipeline.id }}; \ fi @@ -1122,9 +1213,15 @@ dosdp-docs-{{ pipeline.id }}: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ PID }}) --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ --template=$(PATTERNDIR)/dosdp-patterns \ --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ PID }})" \ - --outfile=$(DOCSDIR)/patterns/{{ pipeline.id }}{% if project.repo_url %} \ - --data-location-prefix={{ project.repo_url }}/src/patterns/data/{{ pipeline.id }}{% elif project.github_org and project.repo %} \ - --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/{{ pipeline.id }}{% endif %} + {% if project.repo_url -%} + --data-location-prefix={{ project.repo_url }}/src/patterns/data/{{ pipeline.id }} \ + {% elif project.github_org and project.repo -%} + --data-location-prefix=https://github.com/{{ project.github_org -}} + /{{ project.repo -}} + /tree/{{ project.git_main_branch -}} + /src/patterns/data/{{ pipeline.id }} \ + {% endif -%} + --outfile=$(DOCSDIR)/patterns/{{ pipeline.id }} {% endfor -%} {% endif %}{# !project.pattern_pipelines_group is defined -#} @@ -1147,7 +1244,8 @@ $(PATTERNDIR)/data/{{ pipeline.id }}/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml {% for matches in project.pattern_pipelines_group.matches -%} dosdp-matches-{{ matches.id }}: {{ matches.ontology }} $(ALL_PATTERN_FILES) $(DOSDPT) query {% if matches.dosdp_tools_options %}{{ matches.dosdp_tools_options }} \ - {% endif %}--ontology=$< \ + {% endif -%} + --ontology=$< \ --catalog=$(CATALOG) --reasoner=elk \ --batch-patterns="$(ALL_PATTERN_NAMES)" \ --template="$(PATTERNDIR)/dosdp-patterns" \ @@ -1450,11 +1548,10 @@ $(ONT)-base.owl: $(EDIT_PREPROCESSED) {% if project.release_materialize_object_properties -%} materialize {# -#} {% for iri in project.release_materialize_object_properties -%} - --term {{ iri }}{# -#} + --term {{ iri -}} {% if not loop.last %} \ {# -#} {% else %} \ - {# -#} {% endif -%} {% endfor -%} {% endif -%} @@ -1466,12 +1563,7 @@ $(ONT)-base.owl: $(EDIT_PREPROCESSED) remove {# -#} {% if project.namespaces is not none -%} {% for iri in project.namespaces -%} - --base-iri {{ iri }}{# -#} - {% if not loop.last %} \ - {# -#} - {% else %} \ - {# -#} - {% endif -%} + --base-iri {{ iri }} \ {% endfor -%} {% else -%} --base-iri $(URIBASE)/{{ project.id.upper() }} \ @@ -1518,11 +1610,10 @@ $(ONT)-full.owl: $(EDIT_PREPROCESSED) {% if project.release_materialize_object_properties -%} materialize {# -#} {% for iri in project.release_materialize_object_properties -%} - --term {{ iri }}{# -#} + --term {{ iri -}} {% if not loop.last %} \ {# -#} {% else %} \ - {# -#} {% endif -%} {% endfor -%} {% endif -%} @@ -1824,7 +1915,7 @@ RELEASE_ASSETS = $(MAIN_FILES){# -#} {# This must be kept in sync with what the copy_release_files rule below does -- that is, this should list all files copied by that rule. -#} RELEASED_ASSETS = $(foreach n, $(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){# -#} - {% if project.sssom_mappingset_group.released_products is defined %} + {% if project.sssom_mappingset_group.released_products is defined %} \ $(foreach n, $(RELEASED_MAPPINGS), $(RELEASEDIR)/mappings/$(n).sssom.tsv){# -#} {% endif %} From 4350eb47d37427807c695ed33772d009be127e1f Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 23 Aug 2026 16:20:01 +0100 Subject: [PATCH 21/25] Typo fixes. --- .../odk/templates/src/ontology/Makefile.jinja2 | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index bf9b9c9..c5b0436 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1300,7 +1300,7 @@ $(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) else # PAT=false # Even if pattern generation is disabled, we still extract a seed from definitions.owl $(TMPDIR)/all_pattern_terms.txt: $(PATTERNDIR)/definitions.owl - $(ROBOT) query --intput $< --use-graphs true --format csv \ + $(ROBOT) query --input $< --use-graphs true --format csv \ --query $(SPARQLDIR)/terms.sparql $@ # Dummy target to allow "make test" to skip DOSDP validation under PAT=false @@ -1316,7 +1316,7 @@ endif # ---------------------------------------- SUBSETS ={% for s in project.subset_group.products %} {{ s.id }}{% endfor %} -SUBSET_FILES = $(foreach s, $(SUBSET), $(foreach f, $(sort $(FORMATS) tsv), $(SUBSETDIR)/$(s).$(f))) +SUBSET_FILES = $(foreach s, $(SUBSETS), $(foreach f, $(sort $(FORMATS) tsv), $(SUBSETDIR)/$(s).$(f))) .PHONY: all_subsets all_subsets: $(SUBSET_FILES) @@ -1588,7 +1588,7 @@ $(ONT)-base.owl: $(EDIT_PREPROCESSED) $(ONT)-baselite.owl: $(EDIT_PREPROCESSED) {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - $(ROBOT_SHARED_COMMANDS) \ + $(SHARED_ROBOT_COMMANDS) \ annotate --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} @@ -1636,7 +1636,7 @@ $(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} $(ROBOT_RELEASE_IMPORT_MODE) \ - $(ROBOT_SHARED_COMMANDS) \ + $(SHARED_ROBOT_COMMANDS) \ annotate --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} @@ -1671,7 +1671,7 @@ $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) odk:normalize --base-iri {{ project.uribase }} \ --subset-decls true --synonym-decls true \ repair --merge-axiom-annotations true \ - $(ROBOT_SHARED_COMMANDS) \ + $(SHARED_ROBOT_COMMANDS) \ annotate --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} @@ -1692,7 +1692,7 @@ $(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ {% endif -%} filter --select ontology --term-file $(SIMPLESEED) --trim false \ - $(ROBOT_SHARED_COMMANDS) \ + $(SHARED_ROBOT_COMMANDS) \ annotate --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} @@ -1726,7 +1726,7 @@ $(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) $(KEEPRELATIONS) --select "annotations ontology anonymous self" \ --trim true --signature true \ reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ - $(ROBOT_SHARED_COMMANDS) \ + $(SHARED_ROBOT_COMMANDS) \ annotate --ontology-iri $(ONTBASE)/$@ \ $(ANNOTATE_ONTOLOGY_VERSION) \ {% if project.release_date -%} @@ -1975,7 +1975,7 @@ public_release: @echo "Assets to be pushed:" @ls -alt $(RELEASED_ASSETS) @gh release create $(GHVERSION) - --title "$(VERSION Release" \ + --title "$(VERSION) Release" \ --generate-notes \ $(RELEASED_ASSETS) @@ -2102,7 +2102,7 @@ CURRENT_RELEASE = $(ONTBASE).owl $(TMPDIR)/current-release.owl: wget $(CURRENT_RELEASE) --output-document $@ -$(REPORTDIR)/release-diff: $(ONT).owl $(TMPDIR)/current-release.owl +$(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl $(ROBOT) diff --left $(TMPDIR)/current-release.owl \ --right $(ONT).owl \ --labels true --format markdown --output $@ From ca59ee38556bca40c084ce5f440c971d38681633 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 23 Aug 2026 16:22:07 +0100 Subject: [PATCH 22/25] Restore printing to stdout when validating profile. So that the output can be seen from the log of a GitHub Action run. --- src/incatools/odk/templates/src/ontology/Makefile.jinja2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index c5b0436..165e12c 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -352,7 +352,8 @@ check_for_robot_updates: $(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) $(ROBOT) merge --input $< \ validate-profile --profile DL \ - --output $@ + --output $@ \ + || { cat $@ && exit 1; } .PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt From a6f48214f3e18e12c3b64f148c9f88ba6440530f Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 23 Aug 2026 16:27:42 +0100 Subject: [PATCH 23/25] Write to $(TMPDIR) when normalizing the edit file. Users might want to re-serialize the edit file after they have edited it, but _before_ they have committed their changes to the repository (so as to avoid having to commit it again if it turned out that the file was indeed in need of normalization). This means they can't rely simply on `git restore` to recover from a botched write operation (if the normalization command is interrupted right at the wrong moment). --- src/incatools/odk/templates/src/ontology/Makefile.jinja2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 165e12c..0ea5e91 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -2043,7 +2043,8 @@ clean: .PHONY: normalize_obo_src normalize_obo_src: $(SRC) | all_robot_plugins $(ROBOT) repair --input $< --merge-axiom-annotations true \ - convert --output $< + convert --output $(TMPDIR)/normalized-$< \ + && mv $(TMPDIR)/normalized-$< $< {% endif -%} @@ -2055,7 +2056,8 @@ normalize_src: $(SRC) {% else -%} ofn {# -#} {% endif -%} - --output $< + --output $(TMPDIR)/normalized-$< \ + && mv $(TMPDIR)/normalized-$< $< # Debugging tools # ---------------------------------------- From a4ea06fd65e8f388616bf41a0af0bfe5dd90e043 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 23 Aug 2026 16:33:35 +0100 Subject: [PATCH 24/25] Do not include help for non-existing commands. --- src/incatools/odk/templates/src/ontology/Makefile.jinja2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 0ea5e91..80941d7 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -2188,11 +2188,15 @@ Editor utilities: Additional build commands (advanced users): * all: Run the entire pipeline (like prepare_release), but without copying the release files to the release directory. +{% if project.subset_group is not none -%} * all_subsets: Build all subsets. +{% endif -%} +{% if project.use_mappings -%} +* all_mappings: Update all SSSOM mapping sets. +{% endif -%} * custom_reports: Generate all custom sparql reports you have configured in your project. * all_assets: Build all assets. * show_assets: Print a list of all assets that would be build by the release pipeline. -* all_mappings: Update all SSSOM mapping sets. Additional QC commands (advanced users): * robot_reports: Run all configured ROBOT reports. From 5d6893310f4b720b3fdd37f2541172b5a83a4c43 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Sun, 23 Aug 2026 16:45:46 +0100 Subject: [PATCH 25/25] Include RDF/XML check in the main `test` target. --- src/incatools/odk/templates/src/ontology/Makefile.jinja2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index 80941d7..8f2fe01 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -187,9 +187,10 @@ config_check: {% endif -%} .PHONY: test -test: odkversion validate_idranges {# -#} - {% if project.use_dosdps %}dosdp_validation {% endif -%} +test: odkversion validate_idranges{# -#} + {% if project.use_dosdps %} dosdp_validation{% endif %} \ reason_test sparql_test robot_reports{# -#} + {% if project.ensure_valid_rdfxml %} check_rdxfml_products{% endif -%} {% if project.robot.report.ensure_owl2dl_profile|default(true) %} \ $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{# -#} {% endif %}