diff --git a/src/incatools/odk/model.py b/src/incatools/odk/model.py index a43371c..f209c3a 100644 --- a/src/incatools/odk/model.py +++ b/src/incatools/odk/model.py @@ -1209,3 +1209,7 @@ def derive_fields(self) -> None: or self.public_release == "github_python" ): self.public_release = "github" + + # Exporting to OWL is mandatory, even if not explicitly listed + if "owl" not in self.export_formats: + self.export_formats.append("owl") diff --git a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 index c41c287..8f2fe01 100644 --- a/src/incatools/odk/templates/src/ontology/Makefile.jinja2 +++ b/src/incatools/odk/templates/src/ontology/Makefile.jinja2 @@ -1,124 +1,136 @@ {#- - Jinja2 Template for Makefile This file is a template that generates the Makefile. This comment will not be included. - See: http://jinja.pocoo.org/docs/2.10/templates + See: https://jinja.palletsprojects.com/en/stable/templates/ -#} # ---------------------------------------- # Makefile for {{ project.id }} -# Generated using ontology-development-kit +# Generated using the Ontology Development Kit (ODK) # ODK Version: {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %} # ---------------------------------------- -# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use {{ project.id }}.Makefile instead +# IMPORTANT: DO NOT EDIT THIS FILE. +# To override default make goals, edit {{ project.id }}.Makefile instead. {{ project.custom_makefile_header }} -{%- if project.config_hash %} +{% if project.config_hash -%} # Fingerprint of the configuration file when this Makefile was last generated -CONFIG_HASH= {{ project.config_hash }} -{% endif %} +CONFIG_HASH = {{ project.config_hash }} +{% endif -%} # ---------------------------------------- -# Standard Constants -# ---------------------------------------- -# these can be overwritten on the command line - -OBOBASE= http://purl.obolibrary.org/obo -URIBASE= {{ project.uribase }} -ONT= {{ project.id }} -ONTBASE= {{ project.uribase }}/{% if project.uribase_suffix is not none %}{{ project.uribase_suffix }}{% else %}{{ project.id }}{% endif %} -EDIT_FORMAT= {{ project.edit_format|default('owl') }} -SRC = $(ONT)-edit.$(EDIT_FORMAT) -MAKE_FAST= $(MAKE) IMP=false PAT=false COMP=false MIR=false -CATALOG= {{ project.catalog_file }} -{% if project.use_context -%} -CONTEXT_FILE = config/context.json +# Common variables +# ---------------------------------------- +# These variables are intended to be used throughout the Makefile (and +# the custom {{ project.id }}.Makefile, if needed). They _may_ be +# overriden either on the command line or in the custom Makefile, but in +# most cases this is not recommended. + +# Project settings +OBOBASE = http://purl.obolibrary.org/obo +URIBASE = {{ project.uribase }} +ONT = {{ project.id }} +ONTBASE = {{ project.uribase }}/{% if project.uribase_suffix is not none -%} + {{ project.uribase_suffix -}} + {% else -%} + {{ project.id -}} + {% endif %} +EDIT_FORMAT = {{ project.edit_format|default('owl') }} +FORMATS ={% for format in project.export_formats|sort %} {{ format }}{% endfor %} +RELEASE_ARTEFACTS ={% for release in project.release_artefacts %} {# -#} + {% if release.startswith("custom") -%} + {{ release | replace("custom-", "") -}} + {% else -%} + $(ONT)-{{ release -}} + {% endif -%} + {% endfor %} + +# Directories +RELEASEDIR = ../.. +DOCSDIR = ../../docs +REPORTDIR = reports +TMPDIR = tmp +MIRRORDIR = mirror +IMPORTDIR = imports +SUBSETDIR = subsets +SCRIPTSDIR = ../scripts +SPARQLDIR = ../sparql +COMPONENTSDIR = components +TEMPLATEDIR = ../templates +{% if project.use_dosdps -%} +PATTERNDIR = ../patterns {% endif -%} -ROBOT= robot --catalog $(CATALOG){% if project.use_context %} --add-prefixes $(CONTEXT_FILE){% endif %} -REASONER= {{ project.robot.reasoner }} -{# Kept for backwards compatibility with existing custom Makefiles -#} -{% if project.owltools_memory|length -%} -OWLTOOLS_MEMORY = {{ project.owltools_memory }} +{% if project.use_mappings -%} +MAPPINGDIR = ../mappings {% endif -%} -OWLTOOLS = {% if project.owltools_memory|length %}OWLTOOLS_MEMORY=$(OWLTOOLS_MEMORY) {% endif %}owltools --use-catalog -RELEASEDIR= ../.. -DOCSDIR= ../../docs -REPORTDIR= reports -TEMPLATEDIR= ../templates -TMPDIR= tmp -MIRRORDIR= mirror -IMPORTDIR= imports -SUBSETDIR= subsets -SCRIPTSDIR= ../scripts -UPDATEREPODIR= target -SPARQLDIR = ../sparql -EXTENDED_PREFIX_MAP= $(TMPDIR)/obo.epm.json -COMPONENTSDIR = components -ODK_RESOURCES_DIR ?= /tools/resources -{%- if project.robot.report.custom_profile %} -ROBOT_PROFILE = profile.txt -{%- endif %} -REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} -REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}-l true{% endif %} -REPORT_PROFILE_OPTS = {% if project.robot.report.custom_profile %}--profile $(ROBOT_PROFILE){% endif %} -OBO_FORMAT_OPTIONS = {{ project.robot.obo_format_options }} -SPARQL_VALIDATION_CHECKS = {% for x in project.robot.report.custom_sparql_checks|default(['owldef-self-reference', 'iri-range', 'label-with-iri', 'multiple-replaced_by']) %}{{ x }} {% endfor %} -SPARQL_EXPORTS = {% for x in project.robot.report.custom_sparql_exports|default(['basic-report', 'class-count-by-prefix', 'edges', 'xrefs', 'obsoletes', 'synonyms']) %}{{ x }} {% endfor %} -ODK_VERSION_MAKEFILE = {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %} -RELAX_OPTIONS = {{ project.robot.relax_options }} -REDUCE_OPTIONS = {{ project.robot.reduce_options }} - -TODAY ?= $(shell date +%Y-%m-%d) -OBODATE ?= $(shell date +'%d:%m:%Y %H:%M') -VERSION= $(TODAY) -ANNOTATE_ONTOLOGY_VERSION = annotate -V $(ONTBASE)/releases/$(VERSION)/$@ --annotation owl:versionInfo $(VERSION) -ANNOTATE_CONVERT_FILE = annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) convert -f {{ project.import_component_format|default('ofn') }} --output $@.tmp.owl && mv $@.tmp.owl $@ -OTHER_SRC = {% if project.use_dosdps -%}$(PATTERNDIR)/definitions.owl {% endif -%}{% if project.components is defined -%}{% for component in project.components.products -%}$(COMPONENTSDIR)/{{ component.filename }} {% endfor -%}{% endif %} -ONTOLOGYTERMS = $(TMPDIR)/ontologyterms.txt -EDIT_PREPROCESSED = $(TMPDIR)/$(ONT)-preprocess.owl -{% if project.use_context and 'db' in project.export_formats -%} -CONTEXT_FILE_CSV = $(TMPDIR)/context.csv -{%- endif -%} - -{%- if project.use_dosdps %} -PATTERNDIR= ../patterns -PATTERN_TESTER= dosdp validate -i -DOSDPT= dosdp-tools -PATTERN_RELEASE_FILES= $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl -{% endif %} - -{%- if project.use_mappings %} -MAPPINGDIR= ../mappings -MAPPING_TESTER= sssom validate -SSSOMPY= sssom -MAPPINGS= {% if project.sssom_mappingset_group is not none %}{%- for mapping in project.sssom_mappingset_group.products %}{{ mapping.id }} {% endfor %}{% endif %} -MAPPING_RELEASE_FILES= $(foreach n,$(MAPPINGS), $(MAPPINGDIR)/$(n).sssom.tsv) -{% endif %} {% if project.bridge_group is not none -%} -BRIDGEDIR= bridges +BRIDGEDIR = bridges +{% endif -%} +{% if project.use_translations -%} +TRANSLATIONSDIR= = ../translations +{% endif -%} +ODK_RESOURCES_DIR ?= /tools/resources + +# Files +SRC = $(ONT)-edit.$(EDIT_FORMAT) +OTHER_SRC ={% if project.use_dosdps %} {# -#} + $(PATTERNDIR)/definitions.owl{# -#} + {% endif -%} + {% if project.components is defined -%} + {% for component in project.components.products -%} + {% if not loop.first or project.use_dosdps %} \ + {% endif %} {# -#} + $(COMPONENTSDIR)/{{ component.filename -}} + {% endfor -%} + {% endif %} +CATALOG = {{ project.catalog_file }} +{% if project.use_context -%} +CONTEXT_FILE = config/context.json +{% endif -%} +EXTENDED_PREFIX_MAP = $(TMPDIR)/obo.epm.json +EDIT_PREPROCESSED = $(TMPDIR)/$(ONT)-preprocess.owl +SRCMERGED = $(TMPDIR)/merged-$(ONT)-edit.ofn + +# ROBOT Options +ROBOT = robot --catalog $(CATALOG){% if project.use_context %} --add-prefixes $(CONTEXT_FILE){% endif %} +REASONER = {{ project.robot.reasoner }} +OBO_FORMAT_OPTIONS = {{ project.robot.obo_format_options }} +RELAX_OPTIONS = {{ project.robot.relax_options }} +REDUCE_OPTIONS = {{ project.robot.reduce_options }} + +{# Kept for backward compatibility with existing custom Makefiles -#} +# OWLTools Options +{% if project.owltools_memory|length -%} +OWLTOOLS_MEMORY = {{ project.owltools_memory }} {% endif -%} +OWLTOOLS = {% if project.owltools_memory|length %}OWLTOOLS_MEMORY=$(OWLTOOLS_MEMORY) {% endif %}owltools --use-catalog -{%- if project.use_translations %} -TRANSLATIONSDIR= ../translations -BABELONPY= babelon -q -TRANSLATIONS_OWL= {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}.babelon.owl {% if translation.include_robot_template_synonyms %} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.owl {% endif %}{% endfor %} -TRANSLATIONS_TSV= {%- for translation in project.babelon_translation_group.products %}$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv {% endfor %} -TRANSLATION_FILES= {%- if project.babelon_translation_group is not none %}{% if project.babelon_translation_group.release_merged_translations %}$(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json{% endif %}{% endif %} -{% endif %} +# Date & version +TODAY ?= $(shell date +%Y-%m-%d) +{% if project.release_date -%} +OBODATE ?= $(shell date +'%d:%m:%Y %H:%M') +{% endif -%} +VERSION = $(TODAY) -FORMATS = $(sort {% for format in project.export_formats %} {{ format }}{% endfor %} owl) -FORMATS_INCL_TSV = $(sort $(FORMATS) tsv) -RELEASE_ARTEFACTS = $(sort {% for release in project.release_artefacts %}{% if release.startswith('custom-') %}{{ release | replace("custom-","")}}{% else %}$(ONT)-{{ release }}{% endif %} {% endfor %}) +# Commands +MAKE_FAST = $(MAKE) IMP=false PAT=false COMP=false MIR=false +ANNOTATE_ONTOLOGY_VERSION = annotate --version-iri $(ONTBASE)/releases/$(VERSION)/$@ \ + --annotation owl:versionInfo $(VERSION) +ANNOTATE_CONVERT_FILE = $(ANNOTATE_ONTOLOGY_VERSION) --ontology-iri $(ONTBASE)/$@ \ + convert --format {{ project.import_component_format|default('ofn') }} \ + --output $@ +# Debugging ifeq ($(ODK_DEBUG),yes) -ODK_DEBUG_FILE = debug.log -SHELL = $(SCRIPTSDIR)/run-command.sh +ODK_DEBUG_FILE = debug.log +SHELL = $(SCRIPTSDIR)/run-command.sh endif + # ---------------------------------------- # Workflow control # ---------------------------------------- @@ -142,6 +154,7 @@ COMP = true # Re-generation of pattern-derived files PAT = true + # ---------------------------------------- # Top-level targets # ---------------------------------------- @@ -152,41 +165,57 @@ PAT = true all: all_odk .PHONY: all_odk -all_odk: odkversion{% if project.config_hash %} config_check{% endif %} test custom_reports all_assets{% if project.release_diff %} release_diff{% endif %} +all_odk: odkversion {# -#} + {% if project.config_hash %}config_check {% endif -%} + test custom_reports all_assets{# -#} + {% if project.release_diff %} release_diff{% endif %} -.PHONY: test -test: odkversion validate_idranges {% if project.use_dosdps %}dosdp_validation {% endif %}reason_test sparql_test robot_reports {% if project.robot.report.ensure_owl2dl_profile|default(true) %}$(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{% endif %} - echo "Finished running all tests successfully." +.PHONY: odkversion +odkversion: + @echo "INFO: ODK Makefile {% if env is defined %}{{env['ODK_VERSION'] or "Unknown" }}{% else %}"Unknown"{% endif %}" + @odk-helper info --tools + +{% if project.config_hash -%} +.PHONY: config_check +config_check: + @if [ "$$(tr -d '\r' < $(ONT)-odk.yaml | sha256sum | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \ + echo "INFO: Repository is up-to-date." ; else \ + echo "WARNING: Your ODK configuration has changed since this Makefile was generated." ; \ + echo " You may need to run 'sh run.sh update_repo'." ; \ + fi + +{% endif -%} .PHONY: test +test: odkversion validate_idranges{# -#} + {% if project.use_dosdps %} dosdp_validation{% endif %} \ + reason_test sparql_test robot_reports{# -#} + {% if project.ensure_valid_rdfxml %} check_rdxfml_products{% endif -%} + {% if project.robot.report.ensure_owl2dl_profile|default(true) %} \ + $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt{# -#} + {% endif %} + @echo "INFO: Finished running all tests successfully." + +.PHONY: test_fast test_fast: $(MAKE_FAST) test -.PHONY: release_diff -release_diff: $(REPORTDIR)/release-diff.md - .PHONY: reason_test reason_test: $(EDIT_PREPROCESSED) - $(ROBOT) reason --input $< --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} \ - --exclude-tautologies {{ project.exclude_tautologies }} --output test.owl && rm test.owl - -.PHONY: odkversion -odkversion: - @echo "ODK Makefile $(ODK_VERSION_MAKEFILE)" - @odk-helper info --tools + $(ROBOT) reason --input $< \ + --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} -{%- if project.config_hash %} -.PHONY: config_check -config_check: - @if [ "$$(tr -d '\r' < $(ONT)-odk.yaml | sha256sum | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \ - echo "Repository is up-to-date." ; else \ - echo "Your ODK configuration has changed since this Makefile was generated. You may need to run 'sh run.sh update_repo'." ; fi -{% endif %} +.PHONY: release_diff +release_diff: $(REPORTDIR)/release-diff.md -$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR) -{%- if project.bridge_group is not none %} $(BRIDGEDIR){% endif %}: +$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR){# -#} +{% if project.bridge_group is not none %} $(BRIDGEDIR){# -#} +{% endif %}: mkdir -p $@ + # ---------------------------------------- # ODK-managed ROBOT plugins # ---------------------------------------- @@ -198,125 +227,83 @@ export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins .PHONY: custom_robot_plugins custom_robot_plugins: -{% if project.robot.plugins is not none %} +{% if project.robot.plugins is not none -%} .PHONY: extra_robot_plugins extra_robot_plugins: {% for plugin in project.robot.plugins %} $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar {% endfor %} -{% endif %} + +{% endif -%} # Install all ROBOT plugins to the runtime plugins directory .PHONY: all_robot_plugins all_robot_plugins: $(foreach plugin,$(notdir $(wildcard $(ODK_RESOURCES_DIR)/robot/plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \ - $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \ - custom_robot_plugins {% if project.robot.plugins is not none %}extra_robot_plugins {% endif %} \ + $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \ + custom_robot_plugins {% if project.robot.plugins is not none %}extra_robot_plugins {% endif %} \ # Default rule to install plugins $(ROBOT_PLUGINS_DIRECTORY)/%.jar: @mkdir -p $(ROBOT_PLUGINS_DIRECTORY) - @if [ -f ../../plugins/$*.jar ]; then \ - ln ../../plugins/$*.jar $@ ; \ + @if [ -f ../../plugins/$*.jar ]; then \ + ln ../../plugins/$*.jar $@ ; \ elif [ -f $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar ]; then \ - cp $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar $@ ; \ + cp $(ODK_RESOURCES_DIR)/robot/plugins/$*.jar $@ ; \ fi -# Specific rules for supplementary plugins defined in configuration -{% if project.robot.plugins is not none %}{% for plugin in project.robot.plugins %} +{% if project.robot.plugins is not none -%} +# Additional rules for plugins defined in the ODK configuration +{% for plugin in project.robot.plugins -%} $(ROBOT_PLUGINS_DIRECTORY)/{{ plugin.name }}.jar: -{%- if plugin.mirror_from %} - curl -L -o $@ {{ plugin.mirror_from }} -{%- else %} - echo "ERROR: No URL has been provided for this plugin; you must install it yourself by overwriting this rule in {{ project.id }}.Makefile!" && false -{% endif %} -{% endfor %}{% endif %} - -# ---------------------------------------- -# Release assets -# ---------------------------------------- - -MAIN_PRODUCTS = $(sort $(foreach r,$(RELEASE_ARTEFACTS), $(r)) $(ONT)) -MAIN_GZIPPED = {% if project.gzip_main %}$(foreach f,$(FORMATS), $(ONT).$(f).gz){% endif %} -MAIN_FILES = $(foreach n,$(MAIN_PRODUCTS), $(foreach f,$(FORMATS), $(n).$(f))) $(MAIN_GZIPPED) -SRCMERGED = $(TMPDIR)/merged-$(ONT)-edit.ofn - -.PHONY: all_main -all_main: $(MAIN_FILES) + {# -#} + {% if plugin.mirror_from -%} + curl --location --output $@ {{ plugin.mirror_from -}} + {% else -%} + @echo "ERROR: No URL has been provided for this plugin; you must override" + @echo " this rule in {{ project.id }}.Makefile to install it." + @false{# -#} + {% endif %} +{% endfor -%} +{% endif -%} +{# #} # ---------------------------------------- -# Import assets +# Internal intermediate products # ---------------------------------------- -{% if project.import_group is defined %} -IMPORTS = {% for imp in project.import_group.products %} {{ imp.id }}{% endfor %} -{% else %} -IMPORTS = -{% endif %} -IMPORT_ROOTS = {% if project.import_group.use_base_merging %} $(IMPORTDIR)/merged_import{% else %}$(patsubst %, $(IMPORTDIR)/%_import, $(IMPORTS)){% endif %}{% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import{% endif %} -IMPORT_OWL_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).owl) -{%- if project.import_group.export_obo %} -IMPORT_OBO_FILES = $(foreach n,$(IMPORT_ROOTS), $(n).obo) -IMPORT_FILES = $(IMPORT_OWL_FILES) $(IMPORT_OBO_FILES) - -{% else %} -IMPORT_FILES = $(IMPORT_OWL_FILES) -{% endif %} - -.PHONY: all_imports -all_imports: $(IMPORT_FILES) +{# This section is intended to group the rules that build intermediate + files that are used in several sections throughout the Makefile. -#} -# ---------------------------------------- -# Subset assets -# ---------------------------------------- +# The -edit file plus all components, but without any import module. +$(SRCMERGED): $(EDIT_PREPROCESSED) $(OTHER_SRC) + $(ROBOT) remove --input $< --select imports --trim false \ + merge $(foreach src, $(OTHER_SRC), --input $(src)) \ + convert --format ofn --output $@ -{% if project.subset_group is defined %} -SUBSETS = {% for x in project.subset_group.products %} {{ x.id }}{% endfor %} -{% else %} -SUBSETS = -{% endif %} -SUBSET_ROOTS = $(patsubst %, $(SUBSETDIR)/%, $(SUBSETS)) -SUBSET_FILES = $(foreach n,$(SUBSET_ROOTS), $(foreach f,$(FORMATS_INCL_TSV), $(n).$(f))) +# Override this rule in {{ project.id }}.Makefile to perform any needed +# custom preprocessing on the -edit file. +$(EDIT_PREPROCESSED): $(SRC) + $(ROBOT) convert --input $< --format ofn --output $@ -.PHONY: all_subsets -all_subsets: $(SUBSET_FILES) # ---------------------------------------- -# Mapping assets +# Quality control # ---------------------------------------- -{% if project.sssom_mappingset_group is defined %} -MAPPINGS = {% for x in project.sssom_mappingset_group.products %} {{ x.id }}{% endfor %} -{% if project.sssom_mappingset_group.released_products is defined %} -RELEASED_MAPPINGS = {% for x in project.sssom_mappingset_group.released_products %} {{ x.id }}{% endfor %}{% endif %} -{% else %} -MAPPINGS = -{% endif %} -MAPPING_FILES = $(foreach p, $(MAPPINGS), $(MAPPINGDIR)/$(p).sssom.tsv) -RELEASED_MAPPING_FILES = $(foreach p, $(RELEASED_MAPPINGS), $(MAPPINGDIR)/$(p).sssom.tsv) - -.PHONY: all_mappings -all_mappings: $(MAPPING_FILES) - +# ROBOT reports # ---------------------------------------- -# Bridge assets -# ---------------------------------------- - -{% if project.bridge_group is not none -%} -BRIDGES ={% for bridge in project.bridge_group.products %} {{ bridge.id }}{% endfor %} -BRIDGE_FILES = $(foreach bridge, $(BRIDGES), $(BRIDGEDIR)/$(bridge).owl) - -.PHONY: all_bridges -all_bridges: $(BRIDGE_FILES) - -{% else -%} -BRIDGE_FILES = +OBO_REPORT ={% for x in project.robot.report.report_on|default(["edit"]) -%} + {% if x=="edit" %} $(SRC){% else %} {{ x }}{% endif %}-obo-report{% endfor %} +{% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} +ALIGNMENT_REPORT ={% for x in project.robot.report.report_on|default(["edit"]) -%} + {% if x=="edit" %} $(SRC){% else %} {{ x }}{% endif %}-align-report{% endfor %} {% endif -%} -# ---------------------------------------- -# QC Reports & Utilities -# ---------------------------------------- - -OBO_REPORT = {% for x in project.robot.report.report_on|default(["edit"]) %} {% if x=="edit" %}$(SRC){% else %}{{ x }}{% endif %}-obo-report{% endfor %} -ALIGNMENT_REPORT = {% for x in project.robot.report.report_on|default(["edit"]) %} {% if x =="edit" %}$(SRC){% else %}{{ x }}{% endif %}-align-report{% endfor %} -REPORTS = $(OBO_REPORT){% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology %} $(ALIGNMENT_REPORT){% endif %} -REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) +REPORTS = $(OBO_REPORT){# -#} + {% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology %} {# -#} + $(ALIGNMENT_REPORT){# -#} + {% endif %} +REPORT_FILES = $(foreach report, $(REPORTS), $(REPORTDIR)/$(report).tsv) +REPORT_FAIL_ON = {{ project.robot.report.fail_on|default('None') }} +REPORT_LABEL = {% if project.robot.report.use_labels|default(true) %}--labels true{% endif %} +REPORT_PROFILE_OPTS = {% if project.robot.report.custom_profile %}--profile profile.txt{% endif %} .PHONY: robot_reports robot_reports: $(REPORT_FILES) @@ -324,270 +311,280 @@ robot_reports: $(REPORT_FILES) .PHONY: all_reports all_reports: custom_reports robot_reports -# ---------------------------------------- -# ROBOT OWL Profile checking +$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) + $(ROBOT) report --input $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) \ + --fail-on $(REPORT_FAIL_ON) \ + {% if project.robot.report.use_base_iris -%} + {% if project.namespaces is not none -%} + {% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor -%} + {% else -%}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ + --base-iri $(URIBASE)/{{ project.id }} \ + {% endif -%} + {% endif -%} + --print 5 --output $@ + +{% if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology -%} +$(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins + $(ROBOT) odk:check-align --input $< --reasoner $(REASONER) \ + --upper-ontology-iri {{ project.robot.report.upper_ontology }} \ + {% if project.robot.report.use_base_iris %}--detail BASE-ROOT \ + {% if project.namespaces is not none -%} + {% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor -%} + {% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ \ + --base-iri $(URIBASE)/{{ project.id }} \ + {% endif -%} + {% endif -%} + --report-output $@ + +{% endif -%} + +{% if project.robot.report.custom_profile -%} +check_for_robot_updates: + @odk-helper check-robot-profile + +{% endif -%} + +# ROBOT OWL profile check # ---------------------------------------- # The merge step is necessary to avoid undeclared entity violations. -$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) $(TMPDIR) - $(ROBOT) merge -i $< convert -f ofn -o $(TMPDIR)/validate.ofn - $(ROBOT) validate-profile --profile DL -i $(TMPDIR)/validate.ofn -o $@ || { cat $@ && exit 1; } +$(REPORTDIR)/validate_profile_owl2dl_%.txt: % | $(REPORTDIR) + $(ROBOT) merge --input $< \ + validate-profile --profile DL \ + --output $@ \ + || { cat $@ && exit 1; } .PRECIOUS: $(REPORTDIR)/validate_profile_owl2dl_%.txt validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt - echo "$* profile validation completed." + @echo "INFO: $* profile validation completed." -# ---------------------------------------- -# Sparql queries: Q/C +# SPARQL-based checks and reports # ---------------------------------------- -# these live in the ../sparql directory, and have suffix -violation.sparql -# adding the name here will make the violation check live. - -SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql) - -sparql_test: {% for x in project.robot.report.sparql_test_on|default(["edit"]) %} {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %}{% endfor %} | $(REPORTDIR) -ifneq ($(SPARQL_VALIDATION_QUERIES),) - {% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} - {%- if x=="edit" -%} - {% set input = "$(SRCMERGED)"%} - {%- else -%} - {% set input = x %} - {% endif %} - $(ROBOT) verify -i {{ input }} --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR) - {%- endfor %} -endif +# The SPARQL queries used by those checks live in the ../sparql directory, +# in files whose name is of the form NAME-violation.sparql. Adding the +# NAME in the list below will enable the corresponding check. -# ---------------------------------------- -# ROBOT report -# ---------------------------------------- +{# FIXME: The default list should be computed in the seeding script, not here. -#} +SPARQL_VALIDATION_CHECKS ={% for x in project.robot.report.custom_sparql_checks|default(["owldef-self-reference", "iri-range", "label-with-iri", "multiple-replaced_by"]) %} {{ x }}{% endfor %} -$(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot.report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor -%}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif -%}{% endif -%} --print 5 -o $@ +sparql_test:{% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} + {% if x =="edit" %} $(SRCMERGED){# -#} + {% else %} {{ x -}} + {% endif -%} + {% endfor %} | $(REPORTDIR) +ifneq ($(SPARQL_VALIDATION_CHECKS),) +{% for x in project.robot.report.sparql_test_on|default(["edit"]) -%} +{# #} $(ROBOT) verify --input {% if x=="edit" %}$(SRCMERGED){% else %}{{ x }}{% endif %} \ + --queries $(foreach name, $(SPARQL_VALIDATION_CHECKS), $(SPARQLDIR)/$(name)-violation.sparql) \ + --output-dir $(REPORTDIR) +{% endfor -%} +endif -$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) - $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot.report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif -%} --print 5 -o $@ -{%- if project.robot.report.upper_ontology is defined and project.robot.report.upper_ontology %} - -$(REPORTDIR)/$(SRC)-align-report.tsv: $(SRCMERGED) | $(REPORTDIR) all_robot_plugins - $(ROBOT) odk:check-align -i $< --reasoner $(REASONER) \ - --upper-ontology-iri {{ project.robot.report.upper_ontology }} - {%- if project.robot.report.use_base_iris %} \ - --detail BASE-ROOT \ - {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} \ - {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} \ - {% endif %}{% else %} \ - {% endif %}--report-output $@ +# SPARQL "exports" differ from SPARQL checks in that +# (i) the files containing the queries do not have a '-violation' suffix, +# (ii) they do not trigger an error, they merely report what they found. -$(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins - $(ROBOT) odk:check-align -i $< --reasoner $(REASONER) \ - --upper-ontology-iri {{ project.robot.report.upper_ontology }} - {%- if project.robot.report.use_base_iris %} \ - --detail BASE-ROOT \ - {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} \ - {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} \ - {% endif %}{% else %} \ - {% endif %}--report-output $@ -{%- endif %} +{# FIXME: The default list should be computed in the seeding script, not here. -#} +SPARQL_EXPORTS ={% for x in project.robot.report.custom_sparql_exports|default(["basic-report", "class-count-by-prefix", "edges", "xrefs", "obsoletes", "synonyms"]) %} {{ x }}{% endfor %} -check_for_robot_updates: -{%- if project.robot.report.custom_profile %} - @odk-helper check-robot-profile -{%- else %} - echo "You are not using a custom profile, so you are getting the joy of the latest ROBOT report!" -{% endif %} +.PHONY: custom_reports +custom_reports: $(EDIT_PREPROCESSED) | $(REPORTDIR) +ifneq ($(SPARQL_EXPORTS),) + $(ROBOT) query --input $< --use-graphs true \ + --queries $(foreach name, $(SPARQL_EXPORTS), $(SPARQLDIR)/$(name).sparql) \ + --format tsv --output-dir $(REPORTDIR) +endif + +{% if project.import_group is defined -%} # ---------------------------------------- -# Release assets +# Imports # ---------------------------------------- -ASSETS = \ - $(IMPORT_FILES) \ - $(MAIN_FILES) \{% if project.use_dosdps %} - $(PATTERN_RELEASE_FILES) \{% endif %}{% if project.use_translations %} - $(TRANSLATION_FILES) \{% endif %} - $(REPORT_FILES) \ - $(SUBSET_FILES) \ - $(MAPPING_FILES) \ - $(BRIDGE_FILES) - -{# This does not include mappings (which can optionally be released), - because those are stored in src/mappings, and the fact that this is - one directory _above_ the directory where this Makefile is would - cause problems down the road (notably, it would prevent the use of a - single rsync commmand to copy all release assets except mappings to - the release directory). Basically, this is "all release assets that - are in this src/ontology directory". -#} -RELEASE_ASSETS = \ - $(MAIN_FILES) {% if project.import_group is defined %}{% if project.import_group.release_imports %}$(IMPORT_FILES) {% endif %}{% endif %}\ - $(SUBSET_FILES){% if project.robot.report.release_reports %} \ - $(REPORT_FILES){% endif %}\ - $(BRIDGE_FILES) - -.PHONY: all_assets -all_assets: $(ASSETS) {% if project.ensure_valid_rdfxml %}check_rdfxml_assets{% endif %} - -.PHONY: show_assets -show_assets: - echo $(ASSETS) - du -sh $(ASSETS) - -check_rdfxml_%: % - @odk-helper check-rdfxml {% if project.extra_rdfxml_checks %}--jena --rdflib {% endif %}$< +IMPORTS ={% for imp in project.import_group.products %} {{ imp.id }}{% endfor %} -.PHONY: check_rdfxml_assets -check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl) - -# ---------------------------------------- -# Release Management +# Mirrors # ---------------------------------------- -{% if 'basic' in project.release_artefacts or project.primary_release == 'basic' -%} -KEEPRELATIONS=keeprelations.txt -{% endif -%} - -CLEANFILES=$(MAIN_FILES) $(SRCMERGED) $(EDIT_PREPROCESSED) +ifeq ($(MIR),true) +{% for ont in project.import_group.products -%} -# This should be executed by the release manager whenever time comes to make a release. -# It will ensure that all assets/files are fresh, and will copy to release folder -.PHONY: prepare_release -prepare_release: all_odk - $(MAKE) copy_release_files - rm -f $(CLEANFILES) - @echo "Release files are now in $(RELEASEDIR) - now you should commit, push and make a release \ - on your git hosting site such as GitHub or GitLab" +## ONTOLOGY: {{ ont.id }} +{% if ont.description -%} +## {{ ont.description }} +{% endif -%} -.PHONY: prepare_release_fast -prepare_release_fast: - $(MAKE) prepare_release IMP=false PAT=false MIR=false COMP=false +{% if ont.mirror_type != "no_mirror" -%} +.PHONY: download-mirror-{{ ont.id }} +download-mirror-{{ ont.id }}: | $(TMPDIR) + odk-helper download --output $(TMPDIR)/$@.owl \ + --reference $(MIRRORDIR)/{{ ont.id }}.owl \ + --max-retry {{ project.import_group.mirror_retry_download }} \ + {% if ont.mirror_from -%} + {{ ont.mirror_from -}} + {% elif ont.use_variant is not none -%} + $(OBOBASE)/{{ ont.id }}/{{ ont.id }}-{{ ont.use_variant }}.owl{% if ont.use_gzipped %}.gz{% endif -%} + {% else -%} + $(OBOBASE)/{{ ont.id }}.owl{% if ont.use_gzipped %}.gz{% endif -%} + {% endif %} -# This rule does the bulk of the work for prepare_release, copying all -# files to the release directory. It is mostly intended for internal -# use, by the prepare_release rule itself or by other ODK scripts. -.PHONY: copy_release_files -copy_release_files: - rsync -R $(RELEASE_ASSETS) $(RELEASEDIR){% if project.sssom_mappingset_group is defined %}{% if project.sssom_mappingset_group.released_products is defined %} - mkdir -p $(RELEASEDIR)/mappings - cp -rf $(RELEASED_MAPPING_FILES) $(RELEASEDIR)/mappings{% endif %}{% endif %} +{% if ont.mirror_type == 'custom' -%} +$(MIRRORDIR)/{{ ont.id }}.owl: + @echo "ERROR: You have configured your default mirror type to be custom; + @echo " this behavior needs to be overwritten in {{ project.id }}.Makefile!" + @false +{% else -%} +{% if ont.is_large -%} +ifeq ($(IMP_LARGE),true) +{% endif -%} +$(MIRRORDIR)/{{ ont.id }}.owl: download-mirror-{{ ont.id }} + @if [ -f $(TMPDIR)/download-mirror-{{ ont.id }}.owl ]; then \ + $(ROBOT) + {%- if ont.make_base or "base" == ont.mirror_type -%} + {# #} remove --input $(TMPDIR)/download-mirror-{{ ont.id }}.owl + {%- if ont.base_iris is not none %} \ + {%- for iri in ont.base_iris %} + --base-iri {{ iri }} \ + {%- endfor -%} + {%- else %} \ + --base-iri $(OBOBASE)/{{ ont.id.upper() }} \ + {%- endif %} + --axioms external --preserve-structure false --trim false \ + convert --format ofn --output $@ ; \ + {%- else -%} + {# #} convert --input $(TMPDIR)/download-mirror-{{ ont.id }}.owl \ + --format ofn --output $@ ; \ + {%- endif %} + fi +{% if ont.is_large -%} +endif +{% endif -%} -# All released assets, in their final location within the release -{# ODK devs: keep that in sync with the copy_release_files rule above - -- that is, this should list all files copied by that rule. -#} -RELEASE_ASSETS_AFTER_RELEASE=$(foreach n,$(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){% if project.sssom_mappingset_group is defined %}{% if project.sssom_mappingset_group.released_products is defined %} \ - $(foreach n,$(RELEASED_MAPPINGS), $(RELEASEDIR)/mappings/$(n).sssom.tsv){% endif %}{% endif %} +{% endif -%} +.PRECIOUS: $(MIRRORDIR)/{{ ont.id }}.owl -.PHONY: show_release_assets -show_release_assets: - @echo $(RELEASE_ASSETS_AFTER_RELEASE) +{% endif %}{# !ont.mirror_type != 'no_mirror' -#} -CURRENT_RELEASE=$(ONTBASE).owl +{% endfor %}{# for ont in project.import_group.products -#} -$(TMPDIR)/current-release.owl: - wget $(CURRENT_RELEASE) -O $@ +{% if project.import_group.use_base_merging -%} +ALL_MIRRORS = $(foreach imp, $(IMPORTS), $(MIRRORDIR)/$(imp).owl) +MERGE_MIRRORS = true -$(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl - $(ROBOT) diff --labels true --left $(TMPDIR)/current-release.owl --right $(ONT).owl -f markdown -o $@ +ifeq ($(MERGE_MIRRORS),true) +$(MIRRORDIR)/merged.owl: $(ALL_MIRRORS) + $(ROBOT) merge $(foreach mirror, $^, --input $(mirror)) \ + {%- if project.import_group.annotate_defined_by %} + --annotate-defined-by true \ + {%- endif %} + {%- if project.import_group.base_merge_drop_equivalent_class_axioms %} + remove --axioms equivalent --preserve-structure false \ + {%- endif %} + convert --format ofn --output $@ -# ------------------------ -# Imports: Seeding system -# ------------------------ +.PRECIOUS: $(MIRRORDIR)/merged.owl +endif +{% endif -%} -{% if project.import_group.scan_signature -%} -# pre_seed.txt contains all entities referenced from the -edit file -# and its components -PRESEED=$(TMPDIR)/pre_seed.txt -$(PRESEED): $(SRCMERGED) - $(ROBOT) query --input $< --format --csv \ - --query $(SPARQLDIR)/terms.sparql $@ +endif # MIR=true -{% endif -%} -$(SRCMERGED): $(EDIT_PREPROCESSED) $(OTHER_SRC) - $(ROBOT) remove --input $< --select imports --trim false \ - merge $(foreach src, $(OTHER_SRC), --input $(src)) \ - convert --format ofn --output $@ +{% if project.import_group.scan_signature or project.use_custom_import_module -%} +# Imports seeding system +# ---------------------------------------- -$(EDIT_PREPROCESSED): $(SRC) - $(ROBOT) convert --input $< --format ofn --output $@ +{% if project.import_group.scan_signature -%} +# All entities referenced from the -edit file and any component +PRESEED = $(TMPDIR)/pre_seed.txt +{% endif -%} +{% if project.use_custom_import_module -%} +# Template-defined custom seed +IMPORT_MODULE_SIGNATURE = $(TMPDIR)/external_import_terms.txt +IMPORT_MODULE_TEMPLATE = $(TEMPLATEDIR)/external_import.tsv +{# FIXME: What is the purpose of that by-product? + see https://github.com/INCATools/ontology-development-kit/issues/1365 -#} +IMPORT_MODULE = $(IMPORTDIR)/external_import.owl +{% endif -%} +# All entities to import in addition to defined in the individual _terms.txt files +IMPORTSEED = $(TMPDIR)/seed.txt +T_IMPORTSEED = --term-file $(IMPORTSEED) -{% if 'basic' in project.release_artefacts or 'simple' in project.release_artefacts or 'simple-non-classified' in project.release_artefacts or project.primary_release == 'basic' or project.primary_release == 'simple-non-classified' or project.primary_release == 'simple' -%} -SIMPLESEED=$(TMPDIR)/simple_seed.txt +{% if project.import_group.scan_signature -%} +$(PRESEED): $(SRCMERGED) | $(TMPDIR) + $(ROBOT) query --input $< --format --csv \ + --query $(SPARQLDIR)/terms.sparql $@ -$(SIMPLESEED): $(SRCMERGED) $(ONTOLOGYTERMS) - $(ROBOT) query -f csv -i $< --query ../sparql/simple-seed.sparql $@.tmp &&\ - cat $@.tmp $(ONTOLOGYTERMS) | sort | uniq > $@ &&\ - echo "http://www.geneontology.org/formats/oboInOwl#SubsetProperty" >> $@ &&\ - echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ +{% endif -%} -{% endif -%} -{% if project.use_custom_import_module -%} -IMPORT_MODULE_TEMPLATE=$(TEMPLATEDIR)/external_import.tsv -IMPORT_MODULE_SIGNATURE=$(TMPDIR)/external_import_terms.txt -IMPORT_MODULE=$(IMPORTDIR)/external_import.owl -$(IMPORT_MODULE): $(IMPORT_MODULE_TEMPLATE) | $(TMPDIR) - $(ROBOT) template --template $< {% if project.use_context %}--add-prefixes $(CONTEXT_FILE) {% endif %}\ - --ontology-iri "$(ONTBASE)/external_import.owl" \ - convert -f {{ project.import_component_format|default('ofn') }} \ - --output $@ - -$(IMPORT_MODULE_SIGNATURE): $(IMPORT_MODULE) | $(TMPDIR) - $(ROBOT) query -f csv -i $< --query ../sparql/terms.sparql $@.tmp &&\ - cat $@.tmp | sort | uniq > $@ +{% if project.use_custom_import_module -%} +$(IMPORT_MODULE_SIGNATURE) $(IMPORT_MODULE): $(IMPORT_MODULE_TEMPLATE) | $(TMPDIR) + $(ROBOT) template --template $< \ + --ontology-iri "$(ONTBASE)/external_import.owl" \ + convert --format {{ project.import_component_format|default("ofn") }} \ + --output $(IMPORT_MODULE) \ + query --format csv \ + --query $(SPARQLDIR)/terms.sparql $(IMPORT_MODULE_SIGNATURE) -{% endif -%} +{% endif -%} -{% if project.import_group.scan_signature or project.use_custom_import_module -%} -# seed.txt contains all entities to import in addition to those defined -# in the individual _terms.txt files. -IMPORTSEED = $(TMPDIR)/seed.txt $(IMPORTSEED): -{%- if project.import_group.scan_signature %} $(PRESEED) -{%- if project.use_dosdps %} $(TMPDIR)/all_pattern_terms.txt{% endif -%} -{%- endif -%} -{%-if project.use_custom_import_module %} $(IMPORT_MODULE_SIGNATURE){% endif %} | $(TMPDIR) +{%- if project.import_group.scan_signature %} $(PRESEED) +{%- if project.use_dosdps %} $(TMPDIR)/all_pattern_terms.txt{% endif -%} +{%- endif -%} +{%- if project.use_custom_import_module %} $(IMPORT_MODULE_SIGNATURE){% endif %} | $(TMPDIR) cat $^ | sort | uniq > $@ -T_IMPORTSEED = --term-file $(IMPORTSEED) - -{% endif -%} - -{% if project.import_group is defined -%} -{% if project.import_group.strip_annotation_properties -%} -ANNOTATION_PROPERTIES={% for p in project.import_group.annotation_properties %}{{ p }} {% endfor %} +{% endif %}{# project.import_group.scan_signature or project.use_custom_import_module -#} -{% endif -%} -# ---------------------------------------- # Import modules # ---------------------------------------- -# Most ontologies are modularly constructed using portions of other ontologies -# These live in the imports/ folder -# This pattern uses ROBOT to generate an import module + +IMPORT_ROOTS = {% if project.import_group.use_base_merging -%} + $(IMPORTDIR)/merged_import{# -#} + {% else -%} + $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_import){# -#} + {% endif %} +IMPORT_FILES = $(foreach r, $(IMPORT_ROOTS), $(r).owl){# -#} + {% if project.import_group.export_obo %} \ + $(foreach r, $(IMPORT_ROOTS), $(r).obo){# -#} + {% endif %} ifeq ($(IMP),true) +{% if project.import_group.strip_annotation_properties -%} +ANNOTATION_PROPERTIES ={% for p in project.import_group.annotation_properties %} {{ p }}{% endfor %} +{% endif -%} {% if project.import_group.use_base_merging -%} -ALL_TERMS = $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_terms.txt) +ALL_TERMS = $(foreach imp, $(IMPORTS), $(IMPORTDIR)/$(imp)_terms.txt) -{% if 'slme' == project.import_group.module_type -%} +{% if "slme" == project.import_group.module_type -%} $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ - $(IMPORTSEED) | all_robot_plugins - $(ROBOT) merge --input $<{% if project.import_group.exclude_iri_patterns is not none -%} - {%- for pattern in project.import_group.exclude_iri_patterns %} \ - remove --select "{{ pattern }}"{% endfor %}{% endif %} \ - extract $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ - --force true --copy-ontology-annotations false \ - --individuals {{ project.import_group.slme_individuals }} \ - --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) + $(IMPORTSEED) | all_robot_plugins + $(ROBOT) merge --input $<{# -#} + {% if project.import_group.exclude_iri_patterns is not none -%} + {% for pattern in project.import_group.exclude_iri_patterns %} \ + remove --select "{{ pattern }}"{# -#} + {% endfor -%} + {% endif %} \ + extract $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ + --force true --copy-ontology-annotations false \ + --individuals {{ project.import_group.slme_individuals }} {# -#} + --method {{ project.import_group.module_type_slme }} \ + {% if project.import_group.strip_annotation_properties -%} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + $(foreach f, $(ALL_TERMS), --term-file $(f)) $(T_IMPORTSEED) \ + --select complement --select annotation-properties \ + {% endif -%} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) {% else -%} $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ - $(IMPORTSEED) + $(IMPORTSEED) @echo "ERROR: You have configured the merged import as a custom module;" @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" @false @@ -596,70 +593,72 @@ $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(ALL_TERMS) \ {% else %}{# !project.import_group.use_base_merging -#} ## Default module type ({{ project.import_group.module_type }}) -{% if 'slme' == project.import_group.module_type -%} +{% if "slme" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ - $(IMPORTSEED) | all_robot_plugins + $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --force true --copy-ontology-annotations true \ - --individuals {{ project.import_group.slme_individuals }} \ - --method {{ project.import_group.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'minimal' == project.import_group.module_type -%} + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --force true --copy-ontology-annotations true \ + --individuals {{ project.import_group.slme_individuals }} {# -#} + --method {{ project.import_group.module_type_slme }} \ + {% if project.import_group.strip_annotation_properties -%} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --select complement --select annotation-properties \ + {% endif -%} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) + +{% elif "minimal" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ - $(IMPORTSEED) | all_robot_plugins + $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --force true --copy-ontology-annotations true \ - --method BOT \ - remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ - --axioms external \ - --preserve-structure false --trim false \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement \ - --select "classes individuals annotation-properties" \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'mirror' == project.import_group.module_type -%} + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --force true --copy-ontology-annotations true \ + --method BOT \ + remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ + --axioms external \ + --preserve-structure false --trim false \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --select complement \ + --select "classes individuals annotation-properties" \ + $(ANNOTATE_CONVERT_FILE) + +{% elif "mirror" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - --add-source true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + --add-source true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) -{% elif 'filter' == project.import_group.module_type -%} +{% elif "filter" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl $(IMPORTDIR)/%_terms.txt \ - $(IMPORTSEED) | all_robot_plugins + $(IMPORTSEED) | all_robot_plugins $(ROBOT) merge --input $< \ - annotate --remove-annotations \ - odk:normalize --add-source true \ - remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ - --axioms external \ - --preserve-structure false --trim false \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ - --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ - --select complement \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) - -{% elif 'custom' == project.import_group.module_type -%} + annotate --remove-annotations \ + odk:normalize --add-source true \ + remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" \ + --axioms external \ + --preserve-structure false --trim false \ + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + --term-file $(IMPORTDIR)/$*_terms.txt $(T_IMPORTSEED) \ + --select complement \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) + +{% elif "custom" == project.import_group.module_type -%} $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl @echo "ERROR: You have configured the default module type to be custom;" @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" @@ -681,65 +680,77 @@ $(IMPORTDIR)/%_import.owl: $(MIRRORDIR)/%.owl {% if ont.is_large -%} ifeq ($(IMP_LARGE),true) {% endif -%} -{% if 'slme' == ont.module_type -%} +{% if "slme" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --copy-ontology-annotations true --force true \ - --individuals {{ ont.slme_individuals }} \ - --method {{ ont.module_type_slme }} \{% if project.import_group.strip_annotation_properties %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} - --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement --select annotation-properties \{% endif %} - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'filter' == ont.module_type -%} + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --copy-ontology-annotations true --force true \ + --individuals {{ ont.slme_individuals }} \ + --method {{ ont.module_type_slme }} \ + {% if project.import_group.strip_annotation_properties -%} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + {% for p in ont.annotation_properties -%} + --term {{ p }} \ + {% endfor -%} + --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --select complement --select annotation-properties \ + {% endif -%} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) +{% elif "filter" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --copy-ontology-annotations true --force true --method BOT \ - remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %} - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} - --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'mirror' == ont.module_type -%} + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --copy-ontology-annotations true --force true --method BOT \ + remove --axioms external --preserve-structure false --trim false \ + {%- for iri in ont.base_iris %} + --base-iri {{ iri }} \ + {%- endfor %} + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + {%- for p in ont.annotation_properties %} + --term {{ p }} \ + {%- endfor %} + --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --select complement \ + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) +{% elif "mirror" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --base-iri {{ project.uribase }} \ - --subset-decls true --synonym-decls true \ - --add-source true \ - repair --merge-axiom-annotations true \ - $(ANNOTATE_CONVERT_FILE) -{% elif 'minimal' == ont.module_type -%} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + --add-source true \ + repair --merge-axiom-annotations true \ + $(ANNOTATE_CONVERT_FILE) +{% elif "minimal" == ont.module_type -%} $(IMPORTDIR)/{{ ont.id }}_import.owl: $(MIRRORDIR)/{{ ont.id }}.owl $(IMPORTDIR)/{{ ont.id }}_terms.txt $(IMPORTSEED) | all_robot_plugins $(ROBOT) annotate --input $< --remove-annotations \ - odk:normalize --add-source true \ - extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --copy-ontology-annotations true --force true --method BOT \ - remove --axioms external --preserve-structure false --trim false \{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %} - odk:normalize --base-iri {{ project.uribase }} --subset-decls true \ - --synonym-decls true \ - repair --merge-axiom-annotations true \ - remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \{% for p in ont.annotation_properties %} - --term {{ p }} \{% endfor %} - --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ - --select complement \ - --select "classes individuals annotation-properties" \ - $(ANNOTATE_CONVERT_FILE) + odk:normalize --add-source true \ + extract --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --copy-ontology-annotations true --force true --method BOT \ + remove --axioms external --preserve-structure false --trim false \ + {%- for iri in ont.base_iris %} + --base-iri {{ iri }} \ + {%- endfor %} + odk:normalize --base-iri {{ project.uribase }} --subset-decls true \ + --synonym-decls true \ + repair --merge-axiom-annotations true \ + remove $(foreach p, $(ANNOTATION_PROPERTIES), --term $(p)) \ + {%- for p in ont.annotation_properties %} + --term {{ p }} \ + {%- endfor %} + --term-file $(IMPORTDIR)/{{ ont.id }}_terms.txt $(T_IMPORTSEED) \ + --select complement \ + --select "classes individuals annotation-properties" \ + $(ANNOTATE_CONVERT_FILE) {% elif 'custom' -%} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if 'no_mirror' != ont.mirror_type %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if "no_mirror" != ont.mirror_type %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} @echo "ERROR: You have configured {{ ont.id }} as a custom module;" @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" @false @@ -759,6 +770,10 @@ $(IMPORTDIR)/%_import.obo: $(IMPORTDIR)/%_import.owl {% endif -%} endif # IMP=true +.PHONY: all_imports +all_imports: $(IMPORT_FILES){# -#} + {% if project.orcidio_support %} $(IMPORTDIR)/orcidio_import.owl{% endif %} + .PHONY: refresh-imports refresh-imports: $(MAKE) IMP=true MIR=true PAT=false IMP_LARGE=true clean all_imports @@ -766,8 +781,8 @@ refresh-imports: .PHONY: no-mirror-refresh-imports no-mirror-refresh-imports: $(MAKE) --assume-new=$(SRC) \ - $(foreach imp,$(IMPORTS),--assume-new=$(IMPORTDIR)/$(imp)_terms.txt) \ - IMP=true MIR=false PAT=false IMP_LARGE=true all_imports + $(foreach imp, $(IMPORTS), --assume-new=$(IMPORTDIR)/$(imp)_terms.txt) \ + IMP=true MIR=false PAT=false IMP_LARGE=true all_imports .PHONY: refresh-imports-excluding-large refresh-imports-excluding-large: @@ -777,82 +792,24 @@ refresh-imports-excluding-large: refresh-%: rm -f $(TMPDIR)/download-mirror-$*.owl.info $(MAKE) --assume-new=$(SRC) --assume-new=$(IMPORTDIR)/$*_terms.txt \ - IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl + IMP=true IMP_LARGE=true MIR=true PAT=false $(IMPORTDIR)/$*_import.owl .PHONY: no-mirror-refresh-% no-mirror-refresh-%: $(MAKE) --assume-new=$(SRC) --assume-new=$(IMPORTDIR)/$*_terms.txt \ - IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl - -{% endif %}{# !project.import_group is defined -#} - -{% if project.orcidio_support -%} -# ---------------------------------------- -# ORCIDIO module -# ---------------------------------------- - -# By default, the ORCIDIO module is seeded from ORCID references in ALL -# IRI-valued annotations in the ontology, regardless of the annotation -# property. To seed only from annotations using specific properties, -# override the following variable with the desired list of properties. -ORCIDIO_PROPERTIES = - -ifeq ($(IMP),true) -$(IMPORTDIR)/orcidio_import.owl: $(SRCMERGED) $(MIRRORDIR)/orcidio.owl | all_robot_plugins - $(ROBOT) odk:extract-orcids --input $(SRCMERGED) \ - --orcid-file $(MIRRORDIR)/orcidio.owl \ - $(foreach p, $(ORCIDIO_PROPERTIES),--property $(p)) \ - $(ANNOTATE_CONVERT_FILE) - -endif - -ifeq ($(MIR),true) -.PHONY: download-mirror-orcidio -download-mirror-orcidio: | $(TMPDIR) $(MIRRORDIR) - @odk-helper download --output $(TMPDIR)/$@.owl \ - --reference $(MIRRORDIR)/orcidio.owl{% if project.import_group is defined %} \ - --max-retry {{ project.import_group.mirror_retry_download }}{% endif %} \ - https://w3id.org/orcidio/orcidio.owl + IMP=true IMP_LARGE=true MIR=false PAT=false $(IMPORTDIR)/$*_import.owl -$(MIRRORDIR)/orcidio.owl: download-mirror-orcidio - @if [ -f $(TMPDIR)/download-mirror-orcidio.owl ]; then \ - cp $(TMPDIR)/download-mirror-orcidio.owl $@ ; \ - fi - -endif -{% endif -%} +{% endif %}{# !project.import_group is defined -#} {% if project.components is not none -%} # ---------------------------------------- # Components # ---------------------------------------- -# Some ontologies contain external and internal components. A component is included in the ontology in its entirety. +# Some ontologies contain external and internal components. +# A component is included in the ontology in its entirety. ifeq ($(COMP),true) -.PHONY: all_components -all_components: $(OTHER_SRC) - -.PHONY: recreate-components -recreate-components: - $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ - {% endfor %}COMP=true IMP=false MIR=true PAT=true all_components - -.PHONY: no-mirror-recreate-components -no-mirror-recreate-components: - $(MAKE) {% for component in project.components.products %}--assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ - {% endfor %}COMP=true IMP=false MIR=false PAT=true all_components - -.PHONY: recreate-% -recreate-%: - rm -f $(TMPDIR)/download-component-$*.owl.info - $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ - COMP=true IMP=false MIR=true PAT=true $(COMPONENTSDIR)/$*.owl - -.PHONY: no-mirror-recreate-% -no-mirror-recreate-%: - $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ - COMP=true IMP=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl $(COMPONENTSDIR)/%.owl: $(TMPDIR)/stamp-component-%.owl | $(COMPONENTSDIR) test -f $@ || touch $@ @@ -871,159 +828,281 @@ download-component-{{ component.filename }}: | $(TMPDIR) --reference $(COMPONENTSDIR)/{{ component.filename }} \ {{ component.source }} -$(COMPONENTSDIR)/{{ component.filename }}: download-component-{{ component.filename }} $(TMPDIR)/stamp-component-{{ component.filename }} +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename }} download-component-{{ component.filename }} @if [ -f $(TMPDIR)/download-component-{{ component.filename }} ]; then \ - $(ROBOT) merge -i $(TMPDIR)/download-component-{{ component.filename }} \{% if component.make_base %} - remove {% for iri in component.base_iris %}--base-iri {{ iri }} \ - {% endfor %}--axioms external --preserve-structure false --trim false \{% endif %} - $(ANNOTATE_CONVERT_FILE) ; \ + $(ROBOT) merge --input $(TMPDIR)/download-component-{{ component.filename }} \ + {% if component.make_base -%} + remove {% for iri in component.base_iris %}--base-iri {{ iri }} \ + {% endfor -%} + --axioms external --preserve-structure false --trim false \ + {% endif -%} + $(ANNOTATE_CONVERT_FILE) ; \ fi .PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} endif # MIR=true {% elif component.use_template -%} -$(COMPONENTSDIR)/{{ component.filename }}:{% for template in component.templates %} $(TEMPLATEDIR)/{{ template }}{% endfor %} $(TMPDIR)/stamp-component-{{ component.filename }} - $(ROBOT) template {% if component.template_options is not none %}{{ component.template_options }}{% endif %} \ - {% for template in component.templates %}--template $(TEMPLATEDIR)/{{ template }} \ - {% endfor %}$(ANNOTATE_CONVERT_FILE) +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename -}} + {% for template in component.templates %} \ + $(TEMPLATEDIR)/{{ template -}} + {% endfor %} + $(ROBOT) template {% if component.template_options is not none -%} + {{ component.template_options }} \ + {% endif -%} + {% for template in component.templates -%} + --template $(TEMPLATEDIR)/{{ template }} \ + {% endfor -%} + $(ANNOTATE_CONVERT_FILE) .PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} {% elif component.use_mappings -%} -$(COMPONENTSDIR)/{{ component.filename }}:{% for mapping in component.mappings %} $(MAPPINGDIR)/{{ mapping }}{% endfor %} $(TMPDIR)/stamp-component-{{ component.filename }} | all_robot_plugins +$(COMPONENTSDIR)/{{ component.filename }}: $(TMPDIR)/stamp-component-{{ component.filename -}} + {% for mapping in component.mappings %} \ + $(MAPPINGDIR)/{{ mapping -}} + {% endfor %} \ + | all_robot_plugins $(ROBOT) --add-prefix 'sssom: https://w3id.org/sssom/' \ - --add-prefix 'semapv: http://w3id.org/semapv/vocab/' \ - sssom:inject {% for mapping in component.mappings %}--sssom $(MAPPINGDIR)/{{ mapping }} \ - {% endfor %}--create --direct \ - $(ANNOTATE_CONVERT_FILE) + --add-prefix 'semapv: https://w3id.org/semapv/vocab/' \ + sssom:inject {% for mapping in component.mappings -%} + --sssom $(MAPPINGDIR)/{{ mapping }} \ + {% endfor -%} + --create --direct \ + $(ANNOTATE_CONVERT_FILE) .PRECIOUS: $(COMPONENTSDIR)/{{ component.filename }} {% endif -%} {% endfor -%} endif # COMP=true -{% endif %}{# ! project.components is not none -#} -{% if project.import_group is defined -%} -# ---------------------------------------- -# Mirroring upstream ontologies -# ---------------------------------------- +.PHONY: all_components +all_components: $(OTHER_SRC) -ifeq ($(MIR),true) -{% for ont in project.import_group.products -%} +.PHONY: recreate-components +recreate-components: + $(MAKE) {% for component in project.components.products -%} + --assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ + {% endfor -%} + COMP=true IMP=false MIR=true PAT=true all_components -## ONTOLOGY: {{ ont.id }} -{% if ont.description -%} -## {{ ont.description }} -{% endif -%} +.PHONY: no-mirror-recreate-components +no-mirror-recreate-components: + $(MAKE) {% for component in project.components.products -%} + --assume-new=$(TMPDIR)/stamp-component-{{ component.filename }} \ + {% endfor -%} + COMP=true IMP=false MIR=false PAT=true all_components -{% if ont.mirror_type != 'no_mirror' -%} -.PHONY: download-mirror-{{ ont.id }} -.PRECIOUS: $(MIRRORDIR)/{{ ont.id }}.owl -download-mirror-{{ ont.id }}: | $(TMPDIR) - odk-helper download --output $(TMPDIR)/$@.owl \ - --reference $(MIRRORDIR)/{{ ont.id }}.owl \ - --max-retry {{ project.import_group.mirror_retry_download }} \ - {% if ont.mirror_from %}{{ ont.mirror_from }}{% elif ont.use_variant is not none %}$(OBOBASE)/{{ ont.id }}/{{ ont.id }}-{{ ont.use_variant }}.owl{% if ont.use_gzipped %}.gz{% endif %}{% else %}$(OBOBASE)/{{ ont.id }}.owl{% if ont.use_gzipped %}.gz{% endif %}{% endif %} +.PHONY: recreate-% +recreate-%: + rm -f $(TMPDIR)/download-component-$*.owl.info + $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ + COMP=true IMP=false MIR=true PAT=true $(COMPONENTSDIR)/$*.owl -{% if ont.mirror_type == 'custom' -%} -$(MIRRORDIR)/{{ ont.id }}.owl: - @echo "ERROR: You have configured your default mirror type to be custom; - @echo " this behavior needs to be overwritten in {{ project.id }}.Makefile!" - @false +.PHONY: no-mirror-recreate-% +no-mirror-recreate-%: + $(MAKE) --assume-new=$(TMPDIR)/stamp-component-$*.owl \ + COMP=true IMP=false MIR=false PAT=true $(COMPONENTSDIR)/$*.owl -{% else -%} -{% if ont.is_large -%} -ifeq ($(IMP_LARGE),true) -{% endif -%} -$(MIRRORDIR)/{{ ont.id }}.owl: download-mirror-{{ ont.id }} - @if [ -f $(TMPDIR)/download-mirror-{{ ont.id }}.owl ]; then \ - $(ROBOT) {% if ont.make_base or 'base' == ont.mirror_type %}remove -i $(TMPDIR)/download-mirror-{{ ont.id }}.owl \{% if ont.base_iris is not none %}{% for iri in ont.base_iris %} - --base-iri {{ iri }} \{% endfor %}{% else %} - --base-iri $(OBOBASE)/{{ ont.id.upper() }} \{% endif %} - --axioms external --preserve-structure false --trim false \ - convert {% else %}convert -i $(TMPDIR)/download-mirror-{{ ont.id }}.owl {% endif %}--format ofn -o $@ ; \ - fi -{% if ont.is_large -%} -endif -{% endif -%} +{% endif %}{# ! project.components is not none -#} -{% endif -%} +{% if project.orcidio_support -%} +# ---------------------------------------- +# Special components +# ---------------------------------------- -{% endif %}{# !ont.mirror_type != 'no_mirror' -#} +# ORCIDIO module +# ---------------------------------------- -{% endfor %}{# for ont in project.import_group.products -#} +# By default, the ORCIDIO module is seeded from ORCID references in ALL +# IRI-valued annotations in the ontology, regardless of the annotation +# property. To seed only from annotations using specific properties, +# override the following variable with the desired list of properties. +ORCIDIO_PROPERTIES = -{% if project.import_group.use_base_merging -%} -ALL_MIRRORS = $(patsubst %, $(MIRRORDIR)/%.owl, $(IMPORTS)) -MERGE_MIRRORS = true +ifeq ($(IMP),true) +$(IMPORTDIR)/orcidio_import.owl: $(SRCMERGED) $(MIRRORDIR)/orcidio.owl | all_robot_plugins + $(ROBOT) odk:extract-orcids --input $(SRCMERGED) \ + --orcid-file $(MIRRORDIR)/orcidio.owl \ + $(foreach p, $(ORCIDIO_PROPERTIES), --property $(p)) \ + $(ANNOTATE_CONVERT_FILE) -ifeq ($(MERGE_MIRRORS),true) -$(MIRRORDIR)/merged.owl: $(ALL_MIRRORS) - $(ROBOT) merge $(foreach mirror,$^,-i $(mirror)) {% if project.import_group.annotate_defined_by %}--annotate-defined-by true{% endif %} {% if project.import_group.base_merge_drop_equivalent_class_axioms %}remove --axioms equivalent --preserve-structure false {% endif %}\ - convert --format ofn -o $@ -.PRECIOUS: $(MIRRORDIR)/merged.owl endif -{% endif -%} -endif # MIR=true -{% endif %}{# project.import_group is defined -#} +ifeq ($(MIR),true) +.PHONY: download-mirror-orcidio +download-mirror-orcidio: | $(TMPDIR) $(MIRRORDIR) + @odk-helper download --output $(TMPDIR)/$@.owl \ + --reference $(MIRRORDIR)/orcidio.owl \ + {% if project.import_group is defined -%} + --max-retry {{ project.import_group.mirror_retry_download }} \ + {% endif -%} + https://w3id.org/orcidio/orcidio.owl + +$(MIRRORDIR)/orcidio.owl: download-mirror-orcidio + @if [ -f $(TMPDIR)/download-mirror-orcidio.owl ]; then \ + cp $(TMPDIR)/download-mirror-orcidio.owl $@ ; \ + fi + +endif + + +{% endif %}{# !project.orcidio_support -#} -{% if project.subset_group is defined %} +{% if project.use_mappings or project.bridge_group is not none -%} # ---------------------------------------- -# Subsets +# Mappings and bridges # ---------------------------------------- -$(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl - $(ROBOT) export -i $< --include classes \ - --header "ID [IRI]|LABEL" --format tsv --export $@ -.PRECIOUS: $(SUBSETDIR)/%.tsv -$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins - $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true \ - annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@ -.PRECIOUS: $(SUBSETDIR)/%.owl +{% if project.use_mappings -%} +# SSSOM mappings sets +# ---------------------------------------- -{% if 'obo' in project.export_formats %} -$(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@ -{% endif -%} -{% if 'ttl' in project.export_formats %} -$(SUBSETDIR)/%.ttl: $(SUBSETDIR)/%.owl - $(ROBOT) convert --input $< --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats %} -$(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl - $(ROBOT) convert --input $< --check false -f json -o $@.tmp.json &&\ - mv $@.tmp.json $@ -{% endif -%} +MAPPINGS ={% for m in project.sssom_mappingset_group.products %} {{ m.id }}{% endfor %} +RELEASED_MAPPINGS ={% for m in project.sssom_mappingset_group.released_products %} {{ m.id }}{% endfor %} +MAPPING_FILES = $(foreach m, $(MAPPINGS), $(MAPPINGDIR)/$(m).sssom.tsv) +RELEASED_MAPPING_FILES = $(foreach m, $(RELEASED_MAPPINGS), $(MAPPINGDIR)/$(m).sssom.tsv) +MAPPING_TESTER = sssom validate +SSSOMPY = sssom + +.PHONY: all_mappings +all_mappings: $(MAPPING_FILES) -{% endif %} +validate-sssom-%: + tsvalid $(MAPPINGDIR)/$*.sssom.tsv --comment "#" + $(MAPPING_TESTER) $(MAPPINGDIR)/$*.sssom.tsv -# --------------------------------------------- -# Sparql queries: Table exports / Query Reports -# --------------------------------------------- +validate_mappings: + $(MAKE_FAST) $(foreach n, $(MAPPINGS), validate-sssom-$(n)) -SPARQL_EXPORTS_ARGS = $(foreach V,$(SPARQL_EXPORTS),-s $(SPARQLDIR)/$(V).sparql $(REPORTDIR)/$(V).tsv) -# This combines all into one single command +normalize-sssom-%: + sssom-cli --output $(MAPPINGDIR)/$*.sssom.tsv $(MAPPINGDIR)/$*.sssom.tsv -.PHONY: custom_reports -custom_reports: $(EDIT_PREPROCESSED) | $(REPORTDIR) -ifneq ($(SPARQL_EXPORTS_ARGS),) - $(ROBOT) query -f tsv --use-graphs true -i $< $(SPARQL_EXPORTS_ARGS) +normalize_mappings: + $(MAKE_FAST) $(foreach n, $(MAPPINGS), normalize-sssom-$(n)) + +{% if project.sssom_mappingset_group is not none -%} +{% for mapping in project.sssom_mappingset_group.products -%} +{% if mapping.maintenance == "extract" -%} +{% if project.sssom_mappingset_group.mapping_extractor == "sssom-py" -%} +$(TMPDIR)/{{ mapping.id }}.obographs.json: {{ mapping.source_file }} + $(ROBOT) annotate --input $< \ + --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format json --output $@ + +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: $(TMPDIR)/{{ mapping.id }}.obographs.json + $(SSSOMPY) parse $< --input-format obographs-json {# -#} + {% if mapping.sssom_tool_options %}\ + {{ mapping.sssom_tool_options }} \ + {% endif -%} + --output $@ + +{% elif project.sssom_mappingset_group.mapping_extractor == "sssom-java" -%} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: {{ mapping.source_file }} | all_robot_plugins + $(ROBOT) sssom:xref-extract --input $< --replace \ + --ignore-treat-xrefs --all-xrefs \ + --mapping-file $@ + +{% endif -%} +{% elif mapping.maintenance == "manual" -%} +# This mappingset is manually curated, so we only check that the file actually exists. +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: + test -f $@ + +{% elif mapping.maintenance == "merged" -%} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% for source in mapping.source_mappings %} $(MAPPINGDIR)/{{ source }}.sssom.tsv + {%- if not loop.last %} \ + {% endif -%} + {% endfor %} + sssom-cli --output $@ $^ + +{% elif mapping.maintenance == "mirror" -%} +ifeq ($(MIR),true) +.PHONY: download-sssom-{{ mapping.id }} +download-sssom-{{ mapping.id }}: | $(TMPDIR) + odk-helper download --output $(TMPDIR)/download-sssom-{{ mapping.id }}.sssom.tsv \ + --reference $(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv \ + {{ mapping.mirror_from }} + +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: download-sssom-{{ mapping.id }} + @if [ -f $(TMPDIR)/download-sssom-{{ mapping.id }}.sssom.tsv ]; then \ + cp $(TMPDIR)/download-sssom-{{ mapping.id }}.sssom.tsv $@ ; \ + fi endif +{% elif mapping.maintenance == "custom" -%} +$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: + @echo "ERROR: You have configured {{ mapping.id }} as a custom mapping set;" + @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" + @false + +{% endif -%} +{% endfor -%} +{% endif -%} + +{% endif %}{# !project.use_mappings -#} + +{% if project.bridge_group is not none -%} +# Bridge files +# ---------------------------------------- + +BRIDGES ={% for bridge in project.bridge_group.products %} {{ bridge.id }}{% endfor %} +BRIDGE_FILES = $(foreach b, $(BRIDGES), $(BRIDGEDIR)/$(b).owl) + +.PHONY: all_bridges +all_bridges: $(BRIDGE_FILES) + +{% for bridge in project.bridge_group.products -%} +{% if bridge.bridge_type == "sssom" -%} +$(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ + {% for source in bridge.sources -%} + $(MAPPINGDIR)/{{ source }}.sssom.tsv \ + {% endfor -%} + | $(BRIDGEDIR) all_robot_plugins + $(ROBOT) sssom:inject {% for source in bridge.sources -%} + --sssom $(MAPPINGDIR)/{{ source }}.sssom.tsv \ + {% endfor -%} + --ruleset $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ + --create \ + annotate --ontology-iri $(ONTBASE)/bridges/{{ bridge.id }}.owl \ + --version-iri $(ONTBASE)/releases/$(VERSION)/bridges/{{ bridge.id }}.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ + +{% elif bridge.bridge_type == "custom" -%} +$(BRIDGEDIR)/{{ bridge.id }}.owl: + @echo "ERROR: You have configured {{ bridge.id }} as a custom bridge;" + @echo " This rule needs to be overwritten in {{ project.id}}.Makefile!" + @false + +{% endif -%} +{% endfor -%} +{% endif %}{# !project.bridge_group is not none -#} +{# #} +{% endif -%}{# !project.use_mappings or project.bridge_group is not none -#} + {% if project.use_dosdps -%} # ---------------------------------------- -# DOSDP Templates/Patterns +# DOSDP templates/patterns # ---------------------------------------- -ALL_PATTERN_FILES=$(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml) -ALL_PATTERN_NAMES=$(strip $(patsubst %.yaml,%, $(notdir $(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml)))) +PATTERN_TESTER = dosdp validate -i +DOSDPT = dosdp-tools +PATTERN_RELEASE_FILES = $(PATTERNDIR)/definitions.owl $(PATTERNDIR)/pattern.owl -PATTERN_CLEAN_FILES=../patterns/all_pattern_terms.txt \ - $(DOSDP_OWL_FILES_DEFAULT) $(DOSDP_TERM_FILES_DEFAULT){% if project.pattern_pipelines_group is defined -%} - {% for pipeline in project.pattern_pipelines_group.products %} \ - $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}){% endfor %}{% endif %} +ALL_PATTERN_FILES = $(wildcard $(PATTERNDIR)/dosdp-patterns/*.yaml) +ALL_PATTERN_NAMES = $(strip $(basename $(notdir $(ALL_PATTERN_FILES)))) + +PATTERN_CLEAN_FILES = $(PATTERNDIR)/all_pattern_terms.txt \ + $(DOSDP_OWL_FILES_DEFAULT) $(DOSDP_TERM_FILES_DEFAULT) + {%- if project.pattern_pipelines_group is defined %} + {%- for pipeline in project.pattern_pipelines_group.products %} \ + $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}) + {%- endfor %} + {%- endif %} + +EXTERNAL_PATTERN_URLS = $(strip $(file <$(PATTERNDIR)/dosdp-patterns/external.txt)) +EXTERNAL_PATTERN_FILES = $(notdir $(EXTERNAL_PATTERN_URLS)) .PHONY: pattern_clean pattern_clean: @@ -1031,302 +1110,344 @@ pattern_clean: ifeq ($(PAT),true) -.PHONY: patterns -patterns dosdp: - echo "Validating all DOSDP templates" - $(MAKE) dosdp_validation - echo "Building $(PATTERNDIR)/definitions.owl" - $(MAKE) $(PATTERNDIR)/pattern.owl $(PATTERNDIR)/definitions.owl - -# DOSDP Template Validation - -$(TMPDIR)/pattern_schema_checks: $(ALL_PATTERN_FILES) | $(TMPDIR) - $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns/ && touch $@ +.PHONY: patterns dosdp +patterns dosdp: dosdp_validation $(PATTERNDIR)/pattern.owl $(PATTERNDIR)/definitions.owl .PHONY: pattern_schema_checks -pattern_schema_checks dosdp_validation: $(TMPDIR)/pattern_schema_checks +pattern_schema_checks dosdp_validation: $(TMPDIR)/stamp-pattern-schema-checks -.PHONY: update_patterns -update_patterns: download_patterns - if [ -n "$$(find $(TMPDIR) -type f -path '$(TMPDIR)/dosdp/*.yaml')" ]; then cp -r $(TMPDIR)/dosdp/*.yaml $(PATTERNDIR)/dosdp-patterns; fi +$(TMPDIR)/stamp-pattern-schema-checks: $(ALL_PATTERN_FILES) | $(TMPDIR) + $(PATTERN_TESTER) $(PATTERNDIR)/dosdp-patterns + touch $@ -# This command is a workaround for the absence of -N and -i in wget of alpine (the one ODK depend on now). -# It downloads all patterns specified in external.txt .PHONY: download_patterns -download_patterns: - rm -f $(TMPDIR)/dosdp/*.yaml.1 || true - if [ -s $(PATTERNDIR)/dosdp-patterns/external.txt ]; then wget -i $(PATTERNDIR)/dosdp-patterns/external.txt --backups=1 -P $(TMPDIR)/dosdp; fi - rm -f $(TMPDIR)/dosdp/*.yaml.1 || true +download_patterns: | $(TMPDIR) + mkdir -p $(TMPDIR)/dosdp + $(foreach url, $(EXTERNAL_PATTERN_URLS), \ + odk-helper download --output $(TMPDIR)/dosdp/$(notdir $(url)) \ + --reference $(PATTERNDIR)/dosdp-patterns/$(notdir $(url)) \ + $(url);) -$(PATTERNDIR)/dospd-patterns/%.yml: download_patterns - if cmp -s $(TMPDIR)/dosdp-$*.yml $@ ; then echo "DOSDP templates identical."; else echo "DOSDP templates different, updating." &&\ - cp $(TMPDIR)/dosdp-$*.yml $@; fi +.PHONY: update_patterns +update_patterns: download_patterns + @for f in $$(find $(TMPDIR) -type f -path '$(TMPDIR)/dosdp/*.yaml') ; do \ + cp $$f $(PATTERNDIR)/dosdp-patterns ; + done +$(PATTERNDIR)/dosdp-patterns/%.yaml: download_patterns + @if [ -f $(TMPDIR)/dosdp/$*.yaml ]; then \ + cp $(TMPDIR)/dosdp/$*.yaml $@ ; \ + fi -# DOSDP Template: Pipelines -# Each pipeline gets its own directory structure +# DOSDP pipelines +# ---------------------------------------- +# Each pipeline gets its own directory structure. # DOSDP default pipeline - -DOSDP_TSV_FILES_DEFAULT = $(wildcard $(PATTERNDIR)/data/default/*.tsv) -DOSDP_PATTERN_NAMES_DEFAULT = $(strip $(patsubst %.tsv, %, $(notdir $(DOSDP_TSV_FILES_DEFAULT)))) -DOSDP_OWL_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).ofn) -DOSDP_TERM_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).txt) -DOSDP_YAML_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) +DOSDP_TSV_FILES_DEFAULT = $(wildcard $(PATTERNDIR)/data/default/*.tsv) +DOSDP_PATTERN_NAMES_DEFAULT = $(strip $(basename $(notdir $(DOSDP_TSV_FILES_DEFAULT)))) +DOSDP_OWL_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).ofn) +DOSDP_TERM_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/data/default/$(name).txt) +DOSDP_YAML_FILES_DEFAULT = $(foreach name, $(DOSDP_PATTERN_NAMES_DEFAULT), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) $(DOSDP_OWL_FILES_DEFAULT): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(ALL_PATTERN_FILES) - if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ]; then $(DOSDPT) generate --catalog=$(CATALOG) \ - --infile=$(PATTERNDIR)/data/default/ --template=$(PATTERNDIR)/dosdp-patterns --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ - --ontology=$< {{ project.dosdp_tools_options }} --outfile=$(PATTERNDIR)/data/default; fi + if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] ; then \ + $(DOSDPT) generate --catalog=$(CATALOG) \ + --infile=$(PATTERNDIR)/data/default/ \ + --template=$(PATTERNDIR)/dosdp-patterns \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ + --ontology=$< \ + {% if project.dosdp_tools_options -%} + {{ project.dosdp_tools_options }} \ + {% endif -%} + --outfile=$(PATTERNDIR)/data/default ; \ + fi .PHONY: dosdp-docs-default dosdp-docs-default: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_DEFAULT) $(DOSDP_YAML_FILES_DEFAULT) mkdir -p $(DOCSDIR)/patterns/default - $(DOSDPT) docs {{ project.dosdp_tools_options }} --catalog=$(CATALOG) \ - --ontology=$< \ - --infile=$(PATTERNDIR)/data/default \ - --template=$(PATTERNDIR)/dosdp-patterns \ - --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ - --outfile=$(DOCSDIR)/patterns/default{% if project.repo_url %} \ - --data-location-prefix={{ project.repo_url }}/src/patterns/data/default{% elif project.github_org and project.repo %} \ - --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/default{% endif %} - -{% if project.pattern_pipelines_group is defined %} -{% for pipeline in project.pattern_pipelines_group.products %} + $(DOSDPT) docs {% if project.dosdp_tools_options %}{{ project.dosdp_tools_options }} \ + {% endif %}--catalog=$(CATALOG) \ + --ontology=$< \ + --infile=$(PATTERNDIR)/data/default \ + --template=$(PATTERNDIR)/dosdp-patterns \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_DEFAULT)" \ + {% if project.repo_url -%} + --data-location-prefix={{ project.repo_url }}/src/patterns/data/default \ + {% elif project.github_org and project.repo -%} + --data-location-prefix=https://github.com/{{ project.github_org -}} + /{{ project.repo -}} + /tree/{{ project.git_main_branch -}} + /src/patterns/data/default \ + {% endif -%} + --outfile=$(DOCSDIR)/patterns/default + +{% if project.pattern_pipelines_group is defined -%} +{% for pipeline in project.pattern_pipelines_group.products -%} # DOSDP {{ pipeline.id }} pipeline - -DOSDP_TSV_FILES_{{ pipeline.id.upper() }} = $(wildcard $(PATTERNDIR)/data/{{ pipeline.id }}/*.tsv) -DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }} = $(strip $(patsubst %.tsv, %, $(notdir $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }})))) -DOSDP_OWL_FILES_{{ pipeline.id.upper() }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).ofn) -DOSDP_TERM_FILES_{{ pipeline.id.upper() }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).txt) -DOSDP_YAML_FILES_{{ pipeline.id.upper() }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) - -$(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }}) $(ALL_PATTERN_FILES) - if [ "${DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}}" ]; then $(DOSDPT) generate --catalog=$(CATALOG) \ - --infile=$(PATTERNDIR)/data/{{ pipeline.id }} --template=$(PATTERNDIR)/dosdp-patterns/ --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }})" \ - --ontology=$< {{ pipeline.dosdp_tools_options }} --outfile=$(PATTERNDIR)/data/{{ pipeline.id }}; fi +{% set PID = pipeline.id.upper() -%} + +DOSDP_TSV_FILES_{{ PID }} = $(wildcard $(PATTERNDIR)/data/{{ pipeline.id }}/*.tsv) +DOSDP_PATTERN_NAMES_{{ PID }} = $(strip $(basename $(notdir $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }})))) +DOSDP_OWL_FILES_{{ PID }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ PID }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).ofn) +DOSDP_TERM_FILES_{{ PID }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ PID }}), $(PATTERNDIR)/data/{{ pipeline.id }}/$(name).txt) +DOSDP_YAML_FILES_{{ PID }} = $(foreach name, $(DOSDP_PATTERN_NAMES_{{ PID }}), $(PATTERNDIR)/dosdp-patterns/$(name).yaml) + +$(DOSDP_OWL_FILES_{{ PID }}): $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ PID }}) $(ALL_PATTERN_FILES) + if [ "${DOSDP_PATTERN_NAMES_{{ PID }}}" ] ; then \ + $(DOSDPT) generate --catalog=$(CATALOG) \ + --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ + --template=$(PATTERNDIR)/dosdp-patterns/ \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ PID }})" \ + --ontology=$< \ + {% if pipeline.dosdp_tools_options -%} + {{ pipeline.dosdp_tools_options }} \ + {% endif -%} + --outfile=$(PATTERNDIR)/data/{{ pipeline.id }}; \ + fi .PHONY: dosdp-docs-{{ pipeline.id }} -dosdp-docs-{{ pipeline.id }}: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ pipeline.id.upper() }}) $(DOSDP_YAML_FILES_{{ pipeline.id.upper() }}) +dosdp-docs-{{ pipeline.id }}: $(EDIT_PREPROCESSED) $(DOSDP_TSV_FILES_{{ PID }}) $(DOSDP_YAML_FILES_{{ PID }}) mkdir -p $(DOCSDIR)/patterns/{{ pipeline.id }} - $(DOSDPT) docs {{ pipeline.dosdp_tools_options }} --catalog=$(CATALOG) \ - --ontology=$< \ - --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ - --template=$(PATTERNDIR)/dosdp-patterns \ - --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }})" \ - --outfile=$(DOCSDIR)/patterns/{{ pipeline.id }}{% if project.repo_url %} \ - --data-location-prefix={{ project.repo_url }}/src/patterns/data/{{ pipeline.id }}{% elif project.github_org and project.repo %} \ - --data-location-prefix=https://github.com/{{ project.github_org }}/{{ project.repo }}/tree/{{ project.git_main_branch }}/src/patterns/data/{{ pipeline.id }}{% endif %} -{% endfor -%} -{% endif -%} + $(DOSDPT) docs {% if pipeline.dosdp_tools_options %}{{ pipeline.dosdp_tools_options }} \ + {% endif %}--catalog=$(CATALOG) \ + --ontology=$< \ + --infile=$(PATTERNDIR)/data/{{ pipeline.id }} \ + --template=$(PATTERNDIR)/dosdp-patterns \ + --batch-patterns="$(DOSDP_PATTERN_NAMES_{{ PID }})" \ + {% if project.repo_url -%} + --data-location-prefix={{ project.repo_url }}/src/patterns/data/{{ pipeline.id }} \ + {% elif project.github_org and project.repo -%} + --data-location-prefix=https://github.com/{{ project.github_org -}} + /{{ project.repo -}} + /tree/{{ project.git_main_branch -}} + /src/patterns/data/{{ pipeline.id }} \ + {% endif -%} + --outfile=$(DOCSDIR)/patterns/{{ pipeline.id }} -# Generate template file seeds +{% endfor -%} +{% endif %}{# !project.pattern_pipelines_group is defined -#} + +# Template seeds +# ---------------------------------------- -## Generate template file seeds $(PATTERNDIR)/data/default/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/default/%.tsv - $(DOSDPT) terms --infile=$(word 2, $^) --template=$< --obo-prefixes=true --outfile=$@ + $(DOSDPT) terms --infile=$(PATTERNDIR)/data/default/$*.tsv --template $< \ + --obo-prefixes=true --outfile=$@ -{% if project.pattern_pipelines_group is defined -%} -{% for pipeline in project.pattern_pipelines_group.products %} +{% if project.pattern_pipelines_group is defined -%} +{% for pipeline in project.pattern_pipelines_group.products -%} $(PATTERNDIR)/data/{{ pipeline.id }}/%.txt: $(PATTERNDIR)/dosdp-patterns/%.yaml $(PATTERNDIR)/data/{{ pipeline.id }}/%.tsv - $(DOSDPT) terms --infile=$(word 2, $^) --template=$< --obo-prefixes=true --outfile=$@ -{% endfor %} -{% if project.pattern_pipelines_group.matches is iterable -%}{% for matches in project.pattern_pipelines_group.matches %} + $(DOSDPT) terms --infile=$(PATTERNDIR)/data/{{ pipeline.id }}/$*.tsv --template $< \ + --obo-prefixes=true --outfile=$@ + +{% endfor -%} +{% if project.pattern_pipelines_group.matches is iterable -%} +{% for matches in project.pattern_pipelines_group.matches -%} dosdp-matches-{{ matches.id }}: {{ matches.ontology }} $(ALL_PATTERN_FILES) - $(DOSDPT) query --ontology=$< --catalog=$(CATALOG) --reasoner=elk {{ matches.dosdp_tools_options }} \ - --batch-patterns="$(ALL_PATTERN_NAMES)" --template="$(PATTERNDIR)/dosdp-patterns" --outfile="$(PATTERNDIR)/data/{{ matches.id }}/" -{% endfor %}{% endif -%} -{% endif -%} + $(DOSDPT) query {% if matches.dosdp_tools_options %}{{ matches.dosdp_tools_options }} \ + {% endif -%} + --ontology=$< \ + --catalog=$(CATALOG) --reasoner=elk \ + --batch-patterns="$(ALL_PATTERN_NAMES)" \ + --template="$(PATTERNDIR)/dosdp-patterns" \ + --outfile="$(PATTERNDIR)/data/{{ matches.id }}/" + +{% endfor -%} +{% endif -%} +{% endif -%} # Generating the seed file from all the TSVs. -$(TMPDIR)/all_pattern_terms.txt: $(DOSDP_TERM_FILES_DEFAULT) {% if project.pattern_pipelines_group is defined %} {% for pipeline in project.pattern_pipelines_group.products %} $(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}){% endfor %}{% endif %} $(TMPDIR)/pattern_owl_seed.txt +$(TMPDIR)/all_pattern_terms.txt: $(DOSDP_TERM_FILES_DEFAULT) \ + {% if project.pattern_pipelines_group is defined -%} + {%- for pipeline in project.pattern_pipelines_group.products -%} + {# #}$(DOSDP_TERM_FILES_{{ pipeline.id.upper() }}) \ + {% endfor -%} + {%- endif -%} + {# #}$(TMPDIR)/pattern_owl_seed.txt cat $^ | sort | uniq > $@ $(TMPDIR)/pattern_owl_seed.txt: $(PATTERNDIR)/pattern.owl - $(ROBOT) query --use-graphs true -f csv -i $< --query ../sparql/terms.sparql $@ + $(ROBOT) query --input $< --use-graphs true --format csv \ + --query $(SPARQLDIR)/terms.sparql $@ # Pattern pipeline main targets: the generated OWL files +# ---------------------------------------- -# Create pattern.owl, an ontology of all DOSDP patterns +# An ontology of all DOSDP patterns $(PATTERNDIR)/pattern.owl: $(ALL_PATTERN_FILES) - $(DOSDPT) prototype --obo-prefixes true --template=$(PATTERNDIR)/dosdp-patterns --outfile=$@ - -# Generating the individual pattern modules and merging them into definitions.owl -$(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) {% if project.pattern_pipelines_group is defined %} {% for pipeline in project.pattern_pipelines_group.products %} $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}){% endfor %}{% endif %} - if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] {% if project.pattern_pipelines_group is defined %} {% for pipeline in project.pattern_pipelines_group.products %} || [ "${DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}}" ]{% endfor %}{% endif %} && [ $(PAT) = true ]; then $(ROBOT) merge $(addprefix -i , $^) \ - annotate --ontology-iri $(ONTBASE)/patterns/definitions.owl --version-iri $(ONTBASE)/releases/$(TODAY)/patterns/definitions.owl \ - --annotation owl:versionInfo $(VERSION) -o definitions.ofn && mv definitions.ofn $@; fi + $(DOSDPT) prototype --template=$(PATTERNDIR)/dosdp-patterns \ + --obo-prefixes=true --outfile=$@ + +# The component containing all pattern-derived axioms +$(PATTERNDIR)/definitions.owl: $(DOSDP_OWL_FILES_DEFAULT) + {%- if project.pattern_pipelines_group is defined -%} + {%- for pipeline in project.pattern_pipelines_group.products %} \ + $(DOSDP_OWL_FILES_{{ pipeline.id.upper() }}) + {%- endfor -%} + {%- endif %} + if [ "${DOSDP_PATTERN_NAMES_DEFAULT}" ] + {%- if project.pattern_pipelines_group is defined -%} + {%- for pipeline in project.pattern_pipelines_group.products %} \ + || [ "${DOSDP_PATTERN_NAMES_{{ pipeline.id.upper() }}}" ] + {%- endfor -%} + {%- endif %} ; then \ + $(ROBOT) merge $(addprefix --input , $^) \ + annotate --ontology-iri $(ONTBASE)/patterns/definitions.owl \ + --version-iri $(ONTBASE)/releases/$(TODAY)/patterns/definitions.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ ; \ + fi else # PAT=false # Even if pattern generation is disabled, we still extract a seed from definitions.owl $(TMPDIR)/all_pattern_terms.txt: $(PATTERNDIR)/definitions.owl - $(ROBOT) query --use-graphs true -f csv -i $< --query $(SPARQLDIR)/terms.sparql $@ + $(ROBOT) query --input $< --use-graphs true --format csv \ + --query $(SPARQLDIR)/terms.sparql $@ +# Dummy target to allow "make test" to skip DOSDP validation under PAT=false dosdp_validation: endif -{% endif -%} -{% if project.use_mappings -%} -# ---------------------------------------- -# SSSOM Mapping Files -# ---------------------------------------- -validate-sssom-%: - tsvalid $(MAPPINGDIR)/$*.sssom.tsv --comment "#" - $(MAPPING_TESTER) $(MAPPINGDIR)/$*.sssom.tsv - -validate_mappings: - $(MAKE_FAST) $(foreach n,$(MAPPINGS),validate-sssom-$(n)) - -normalize-sssom-%: - sssom-cli --output $(MAPPINGDIR)/$*.sssom.tsv $(MAPPINGDIR)/$*.sssom.tsv - -normalize_mappings: - $(MAKE_FAST) $(foreach n,$(MAPPINGS),normalize-sssom-$(n)) +{% endif %}{# !project.use_dosdps -#} -{% if project.sssom_mappingset_group is not none -%} -{% for mapping in project.sssom_mappingset_group.products -%} -{% if mapping.maintenance == "extract" -%} -{% if project.sssom_mappingset_group.mapping_extractor == "sssom-py" -%} -$(TMPDIR)/{{ mapping.id }}.obographs.json: {{ mapping.source_file }} - $(ROBOT) annotate --input $< \ - --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false --format json --output $@ +{% if project.subset_group is not none -%} +# ---------------------------------------- +# Subsets +# ---------------------------------------- -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: $(TMPDIR)/{{ mapping.id }}.obographs.json - $(SSSOMPY) parse $< -I obographs-json {{ mapping.sssom_tool_options }} -o $@ +SUBSETS ={% for s in project.subset_group.products %} {{ s.id }}{% endfor %} +SUBSET_FILES = $(foreach s, $(SUBSETS), $(foreach f, $(sort $(FORMATS) tsv), $(SUBSETDIR)/$(s).$(f))) -{% elif project.sssom_mappingset_group.mapping_extractor == "sssom-java" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: {{ mapping.source_file }} | all_robot_plugins - $(ROBOT) sssom:xref-extract --input $< --replace \ - --ignore-treat-xrefs --all-xrefs \ - --mapping-file $@ - -{% endif -%} -{% elif mapping.maintenance == "manual" -%} -# This mappingset is manually curated, so we only check that the file actually exists. -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: - test -f $@ +.PHONY: all_subsets +all_subsets: $(SUBSET_FILES) -{% elif mapping.maintenance == "merged" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv:{% for source in mapping.source_mappings %} $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} - sssom-cli --output $@ $^ +$(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl + $(ROBOT) export --input $< --include classes \ + --header "ID [IRI]|LABEL" --format tsv --export $@ +.PRECIOUS: $(SUBSETDIR)/%.tsv -{% elif mapping.maintenance == "mirror" -%} -ifeq ($(MIR),true) -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: - wget "{{ mapping.mirror_from }}" -O $@ -endif +$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins + $(ROBOT) odk:subset --input $< --subset $* --fill-gaps true \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) --output $@ +.PRECIOUS: $(SUBSETDIR)/%.owl -{% elif mapping.maintenance == "custom" -%} -$(MAPPINGDIR)/{{ mapping.id }}.sssom.tsv: - @echo "ERROR: You have configured {{ mapping.id }} as a custom mapping set;" - @echo " This rule needs to be overwritten in {{ project.id }}.Makefile!" - @false +{% if 'obo' in project.export_formats -%} +$(SUBSETDIR)/%.obo: $(SUBSETDIR)/%.owl + $(ROBOT) convert --input $< --check false --format obo \ + $(OBO_FORMAT_OPTIONS) --output $@ -{% endif -%} -{% endfor -%} {% endif -%} +{% if 'ttl' in project.export_formats -%} +$(SUBSETDIR)/%.ttl: $(SUBSETDIR)/%.owl + $(ROBOT) convert --input $< --check false --format ttl --output $@ -{% endif %}{# !project.use_mappings -#} - -{% if project.bridge_group is not none -%} -# ---------------------------------------- -# Bridge files -# ---------------------------------------- - -{% for bridge in project.bridge_group.products -%} -{% if bridge.bridge_type == "sssom" -%} -$(BRIDGEDIR)/{{ bridge.id }}.owl: $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt{% for source in bridge.sources %} \ - $(MAPPINGDIR)/{{ source }}.sssom.tsv{% endfor %} \ - | $(BRIDGEDIR) all_robot_plugins - $(ROBOT) sssom:inject {% for source in bridge.sources %}--sssom $(MAPPINGDIR)/{{ source }}.sssom.tsv \ - {% endfor %}--ruleset $(SCRIPTSDIR)/bridge-{{ bridge.ruleset }}.sssomt \ - --create \ - annotate --ontology-iri $(ONTBASE)/bridges/{{ bridge.id }}.owl \ - --version-iri $(ONTBASE)/releases/$(VERSION)/bridges/{{ bridge.id }}.owl \ - --annotation owl:versionInfo $(VERSION) \ - convert -f ofn -o $@ - -{% elif bridge.bridge_type == "custom" -%} -$(BRIDGEDIR)/{{ bridge.id }}.owl: - @echo "ERROR: You have configured {{ bridge.id }} as a custom bridge;" - @echo " This rule needs to be overwritten in {{ project.id}}.Makefile!" - @false +{% endif -%} +{% if 'json' in project.export_formats -%} +$(SUBSETDIR)/%.json: $(SUBSETDIR)/%.owl + $(ROBOT) convert --input $< --check false --format json --output $@ -{% endif -%} -{% endfor -%} -{% endif %}{# !project.bridge_group is not none -#} +{% endif -%} +{# #} +{% endif %}{# !project.subset_group is defined -#} {% if project.use_translations -%} # ---------------------------------------- -# Babelon Translation Files -# ---------------------------------------- - -{%- if project.babelon_translation_group is not none %} - -TRANSLATIONS_ADAPTER={{ project.babelon_translation_group.oak_adapter|default('pronto:$(ONT).obo') }} -TRANSLATIONS_ONTOLOGY={{ project.babelon_translation_group.translate_ontology|default('$(ONT).obo') }} -TRANSLATE_PREDICATES={% for predicate_id in project.babelon_translation_group.predicates|default(['IAO:0000115', 'rdfs:label'], true) %}{{ predicate_id }} {% endfor %} - -{% for translation in project.babelon_translation_group.products %} -{% if translation.maintenance == "mirror" %} +# Babelon translation files +# ---------------------------------------- + +BABELONPY = babelon -q +{% if project.babelon_translation_group is not none -%} +TRANSLATIONS_OWL ={% for translation in project.babelon_translation_group.products %} $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.owl + {%- if translation.include_robot_template_synonyms %} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.owl{% endif %} + {%- if not loop.last %} \ + {% endif %}{% endfor %} +TRANSLATIONS_TSV ={% for translation in project.babelon_translation_group.products %} $(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv + {%- if not loop.last %} \ + {% endif %}{% endfor %} +TRANSLATION_FILES ={% if project.babelon_translation_group.release_merged_translations %} + {#- #} $(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv $(TRANSLATIONSDIR)/$(ONT)-all.babelon.json + {%- endif %} +TRANSLATIONS_ADAPTER = {{ project.babelon_translation_group.oak_adapter|default("pronto:$(ONT).obo") }} +TRANSLATIONS_ONTOLOGY = {{ project.babelon_translation_group.translate_ontology|default("$(ONT).obo") }} +TRANSLATE_PREDICATES ={% for predicate_id in project.babelon_translation_group.predicates|default(["IAO:0000115", "rdfs:label"], true) %} + {#- #} {{ predicate_id }}{% endfor %} + +{% for translation in project.babelon_translation_group.products -%} +{% if translation.maintenance == "mirror" -%} $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv: wget "{{ translation.mirror_babelon_from }}" -O $@ -{% if translation.include_robot_template_synonyms %} + +{% if translation.include_robot_template_synonyms -%} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.tsv: wget "{{ translation.mirror_synonyms_from }}" -O $@ -{% endif %} -{% else %} + +{% endif -%} +{% else -%} # This mappingset is manually curated, so we only check that the file actually exists. $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv: test -f $@ -{% if translation.include_robot_template_synonyms %} + +{% if translation.include_robot_template_synonyms -%} $(TRANSLATIONSDIR)/{{ translation.id }}.synonyms.tsv: test -f $@ -{% endif %} -{% endif %} - -$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv{% if translation.auto_translate %} - @if [ -z "$(OPENAI_API_KEY)" ]; then echo "OPENAI_API_KEY must be set as as part of the make command, e.g. sh run.sh make OPENAI_API_KEY=\"sk-123\" my_command" && exit 1; fi{% endif %} - $(BABELONPY) prepare-translation $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv \ - --oak-adapter $(TRANSLATIONS_ADAPTER) \ - --language-code {{ translation.language|default('en') }} \ - $(foreach n,$(TRANSLATE_PREDICATES), --field $(n)) \ - --output-source-changed $(TRANSLATIONSDIR)/{{ translation.id }}-changed.babelon.tsv \ - --output-not-translated $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv \ - --include-not-translated {{ translation.include_not_translated|default('false')|lower }} \ - --update-translation-status {{ translation.update_translation_status|default('true')|lower }} \ - --drop-unknown-columns {{ translation.drop_unknown_columns|default('true')|lower }} \ - -o $@{% if translation.auto_translate %} - echo "Warning: By default, the toolkit employs LLM-mediated translations using the OpenAI API. This default may change at any time" - echo "Warning: Never store API keys or other secrets in Makefiles or scripts you have in version control." - export OPENAI_API_KEY="$(OPENAI_API_KEY)" &&\ - $(BABELONPY) translate $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv -o $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv - $(BABELONPY) merge $(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv -o $@ - {%- endif %} - -{% endfor %} -{%- endif %} + +{% endif -%} +{% endif -%} + +$(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv: $(TRANSLATIONS_ONTOLOGY) $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv +{% if translation.auto_translate -%} +{# #} @if [ -z "$(OPENAI_API_KEY)" ] ; then \ + echo "ERROR: OPENAI_API_KEY must be set as as part of the make command," ; \ + echo " e.g. sh run.sh make OPENAI_API_KEY=\"sk-123\" my_command." ; \ + false ; \ + fi +{% endif -%} +{# #} $(BABELONPY) prepare-translation $(TRANSLATIONSDIR)/{{ translation.id }}.babelon.tsv \ + --oak-adapter $(TRANSLATIONS_ADAPTER) \ + --language-code {{ translation.language|default("en") }} \ + $(foreach n, $(TRANSLATE_PREDICATES), --field $(n)) \ + --output-source-changed $(TRANSLATIONSDIR)/{{ translation.id }}-changed.babelon.tsv \ + --output-not-translated $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv \ + --include-not-translated {{ translation.include_not_translated|default("false")|lower }} \ + --update-translation-status {{ translation.update_translation_status|default("true")|lower }} \ + --drop-unknown-columns {{ translation.drop_unknown_columns|default("true")|lower }} \ + -o $@ +{% if translation.auto_translate -%} +{# #} @echo "Warning: By default, the toolkit employs LLM-mediated translations" + @echo " using the OpenAI API. This default may change at any time." + @echo "Warning: Never store API keys or other secrets in Makefiles or" + @echo " scripts you have in version control." + export OPENAI_API_KEY="$(OPENAI_API_KEY)" && \ + $(BABELONPY) translate $(TRANSLATIONSDIR)/{{ translation.id }}-not-translated.babelon.tsv \ + -o $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv + $(BABELONPY) merge $(TRANSLATIONSDIR)/{{ translation.id }}-preprocessed.babelon.tsv \ + $(TRANSLATIONSDIR)/{{ translation.id }}-translated.babelon.tsv \ + -o $@ +{% endif -%} +{# #} +{% endfor -%} +{% endif %}{# !project.babelon_translation_group is not none -#} $(TRANSLATIONSDIR)/%.synonyms.owl: $(TRANSLATIONSDIR)/%.synonyms.tsv $(ROBOT) template --template $< \ - annotate \ - --ontology-iri $(ONTBASE)/translations/$*.synonyms.owl \ - -V $(ONTBASE)/releases/$(VERSION)/translations/$*.synonyms.owl \ - --annotation owl:versionInfo $(VERSION) \ - convert --format ofn --output $@ + annotate --ontology-iri $(ONTBASE)/translations/$*.synonyms.owl \ + --version-iri $(ONTBASE)/releases/$(VERSION)/translations/$*.synonyms.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ .PRECIOUS: $(TRANSLATIONSDIR)/%.synonyms.owl -$(TRANSLATIONSDIR)/%.babelon.owl: $(TRANSLATIONSDIR)/%-preprocessed.babelon.tsv - $(BABELONPY) convert $< --output-format owl -o $@.tmp - $(ROBOT) merge -i $@.tmp \ - annotate \ - --ontology-iri $(ONTBASE)/translations/$*.babelon.owl \ - -V $(ONTBASE)/releases/$(VERSION)/translations/$*.babelon.owl \ - --annotation owl:versionInfo $(VERSION) \ - convert --format ofn --output $@ - @rm $@.tmp +$(TMPDIR)/%.babelon.owl: $(TRANSLATIONSDIR)/%-preprocessed.babelon.tsv | $(TMPDIR) + $(BABELONPY) convert $< --output-format owl -o $@ + +$(TRANSLATIONSDIR)/%.babelon.owl: $(TMPDIR)/%.babelon.owl + $(ROBOT) merge --input $< \ + annotate --ontology-iri $(ONTBASE)/translations/$*.babelon.owl \ + --version-iri $(ONTBASE)/releases/$(VERSION)/translations/$*.babelon.owl \ + --annotation owl:versionInfo $(VERSION) \ + convert --format ofn --output $@ .PRECIOUS: $(TRANSLATIONSDIR)/%.babelon.owl $(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv: $(TRANSLATIONS_TSV) @@ -1335,227 +1456,532 @@ $(TRANSLATIONSDIR)/$(ONT)-all.babelon.tsv: $(TRANSLATIONS_TSV) $(TRANSLATIONSDIR)/%.babelon.json: $(TRANSLATIONSDIR)/%.babelon.tsv $(BABELONPY) convert $< --output-format json -o $@ -{% endif -%} -# ---------------------------------------- -# Release artefacts: export formats -# ---------------------------------------- +{% endif %}{# !project.use_translations -#} + +# ---------------------------------------- +# Release artefacts +# ---------------------------------------- + +SHARED_ROBOT_COMMANDS ={% if project.remove_owl_nothing %} remove --term owl:Nothing{% endif %} +{# Merge all OWL imports (both ODK import modules and components). -#} +ROBOT_RELEASE_IMPORT_MODE = {% if project.use_edit_file_imports -%} + $(ROBOT) merge --input $<{# -#} + {% if project.import_group is not none and + project.import_group.use_base_merging is false and + project.import_group.annotate_defined_by -%} + {# #} --annotate-defined-by true{# -#} + {% endif -%} + {% else -%} + $(ROBOT) remove --input $< --select imports --trim false{# -#} + {% if project.use_dosdps or project.components is not none %} \ + merge $(foreach src, $(OTHER_SRC), --input $(src)){# -#} + {% endif -%} + {% if project.import_group is not none %} \ + merge $(foreach imp, $(IMPORT_FILES), --input $(imp)){# -#} + {% endif -%} + {% endif %} +{# Merge components only. + This is for the release artefacts that start from "base". -#} +ROBOT_RELEASE_IMPORT_MODE_BASE = $(ROBOT) remove --input $< --select imports --trim false{# -#} + {% if project.use_dosdps or project.components is not none %} \ + merge $(foreach src, $(OTHER_SRC), --input $(src)){# -#} + {% endif %} + +MAIN_PRODUCTS = $(sort $(foreach a, $(RELEASE_ARTEFACTS), $(a)) $(ONT)) +MAIN_FILES = $(foreach p, $(MAIN_PRODUCTS), $(foreach f, $(FORMATS), $(p).$(f))){# -#} + {% if project.gzip_main %} \ + $(foreach f, $(FORMATS), $(ONT).$(f).gz){# -#} + {% endif %} -{% for r in project.release_artefacts -%} -{% if r.startswith('custom-') %}{% set release = r | replace("custom-","") %}{% else %}{% set release = "$(ONT)-" ~ r %}{% endif -%} -{% if 'obo' in project.export_formats -%} -{{ release }}.obo: {{ release }}.owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@ -{% endif -%} -{% if 'ttl' in project.export_formats -%} -{{ release }}.ttl: {{ release }}.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ -{% endif -%} -{% if 'json' in project.export_formats -%} -{{ release }}.json: {{ release }}.owl - $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f json -o $@.tmp.json &&\ - mv $@.tmp.json $@ -{% endif -%} -{% endfor -%} +.PHONY: all_main +all_main: $(MAIN_FILES){# -#} + {% if project.ensure_valid_rdfxml %} check_rdfxml_products{% endif %} -{% if 'db' in project.export_formats -%} -{% if project.use_context -%} -$(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) - @odk-helper context2csv $< > $@ -{% endif -%} +{% if project.ensure_valid_rdfxml -%} +.PHONY: check_rdfxml_products +check_rdfxml_products: $(foreach product, $(MAIN_PRODUCTS), check_rdfxml_$(product).owl) -%.db: %.owl{% if project.use_context %} $(CONTEXT_FILE_CSV){% endif %} - @rm -f $*.db $*-relation-graph.tsv.gz .template.db .template.db.tmp - semsql make $*.db{% if project.use_context %} -P $(CONTEXT_FILE_CSV){% endif %} - @rm -f $*-relation-graph.tsv.gz .template.db .template.db.tmp - @test -f $*.db || (echo "SQLite/SemSQL generation failed" && exit 1) -{% endif -%} +check_rdfxml_%: % + @odk-helper check-rdfxml {% if project.extra_rdfxml_checks %}--jena --rdflib {% endif %}$< -# ---------------------------------------- -# Release artefacts: main release artefacts +{% endif -%} +# Release artefacts: main pipelines # ---------------------------------------- -$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl - $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert -o $@.tmp.owl && mv $@.tmp.owl $@ - -{% if 'obo' in project.export_formats -%} -$(ONT).obo: $(ONT).owl - $(ROBOT) convert --input $< --check false -f obo $(OBO_FORMAT_OPTIONS) -o $@ -{% endif -%} -{% if 'ttl' in project.export_formats -%} -$(ONT).ttl: $(ONT).owl - $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f ttl -o $@.tmp.ttl && mv $@.tmp.ttl $@ +{% if "basic" in project.release_artefacts or project.primary_release == "basic" -%} +KEEPRELATIONS = keeprelations.txt {% endif -%} -{% if 'json' in project.export_formats -%} -$(ONT).json: $(ONT).owl - $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - convert --check false -f json -o $@.tmp.json &&\ - mv $@.tmp.json $@ -{% endif -%} - -# ----------------------------------------------------- -# Release artefacts: variants (base, full, simple, etc) -# ----------------------------------------------------- -SHARED_ROBOT_COMMANDS = {% if project.remove_owl_nothing -%}remove --term owl:Nothing{% endif %} - -$(ONTOLOGYTERMS): $(SRCMERGED) - $(ROBOT) query -f csv -i $< --query ../sparql/{{ project.id }}_terms.sparql $@ +{% if "basic" in project.release_artefacts + or "simple" in project.release_artefacts + or "simple-non-classified" in project.release_artefacts + or project.primary_release == "basic" + or project.primary_release == "simple" + or project.primary_release == "simple-non-classified" -%} +SIMPLESEED = $(TMPDIR)/simple_seed.txt + +$(SIMPLESEED): $(SRCMERGED) + $(ROBOT) query --input $< --format csv \ + --query $(SPARQLDIR)/simple-seed.sparql $@.1.tmp \ + --query $(SPARQLDIR)/{{ project.id }}_terms.sparql $@.2.tmp + cat $@.1.tmp $@.2.tmp | sort | uniq > $@ + echo "http://www.geneontology.org/formats/oboInOwl#SubsetProperty" >> $@ + echo "http://www.geneontology.org/formats/oboInOwl#SynonymTypeProperty" >> $@ -{% for format in project.export_formats -%} -{% if project.gzip_main -%} -$(ONT).{{ format }}.gz: $(ONT).{{ format }} - gzip -c $< > $@.tmp && mv $@.tmp $@ -{% endif -%} -{% endfor -%} -{% if 'owl' not in project.export_formats -%} -{% if project.gzip_main -%} -$(ONT).owl.gz: $(ONT).owl - gzip -c $< > $@.tmp && mv $@.tmp $@ {% endif -%} + +# The primary release artefact. $(ONT).owl: $(ONT)-{{ project.primary_release }}.owl - cp $< $@ -{% endif -%} + $(ROBOT) annotate --input $< --ontology-iri $(URIBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) --output $@ + +{% if "base" in project.release_artefacts or project.primary_release == "base" -%} +# Base: A version of the ontology in which axioms outside of the +# ontology's namespace(s) are forcibly removed. +$(ONT)-base.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + {% if project.release_use_reasoner -%} + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default("false")|lower }} \ + {% if project.release_materialize_object_properties -%} + materialize {# -#} + {% for iri in project.release_materialize_object_properties -%} + --term {{ iri -}} + {% if not loop.last %} \ + {# -#} + {% else %} \ + {% endif -%} + {% endfor -%} + {% endif -%} + {% endif -%} + relax $(RELAX_OPTIONS) \ + {% if project.release_use_reasoner -%} + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + {% endif -%} + remove {# -#} + {% if project.namespaces is not none -%} + {% for iri in project.namespaces -%} + --base-iri {{ iri }} \ + {% endfor -%} + {% else -%} + --base-iri $(URIBASE)/{{ project.id.upper() }} \ + {% endif -%} + --axioms external --preserve-structure false --trim false \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ + --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ -# ROBOT pipeline that merges imports, including components. -ROBOT_RELEASE_IMPORT_MODE={% if project.use_edit_file_imports -%}$(ROBOT) merge --input $< {% if project.import_group is defined and project.import_group.use_base_merging is false and project.import_group.annotate_defined_by %}--annotate-defined-by true{% endif -%}{% else -%} -$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} -merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %}{% if project.import_group is defined -%} -merge $(patsubst %, -i %, $(IMPORT_FILES)){% endif %}{% endif %} - -# ROBOT pipeline that removes imports, then merges components. This is for release artefacts that start from "base" -ROBOT_RELEASE_IMPORT_MODE_BASE=$(ROBOT) remove --input $< --select imports --trim false {% if project.use_dosdps or project.components is defined -%} -merge $(patsubst %, -i %, $(OTHER_SRC)) {% endif %} - -{% if 'base' in project.release_artefacts or project.primary_release == 'base' -%} -# base: A version of the ontology that does not include any externally imported axioms. -$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% if project.release_materialize_object_properties is defined and project.release_materialize_object_properties %} - materialize {% for iri in project.release_materialize_object_properties %}--term {{iri}} {% endfor %} \{% endif %}{% endif %} - relax $(RELAX_OPTIONS) \{% if project.release_use_reasoner %} - reduce -r $(REASONER) $(REDUCE_OPTIONS) \{% endif %} - remove {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ - $(SHARED_ROBOT_COMMANDS) \ - annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \ - --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} -{% if 'baselite' in project.release_artefacts or project.primary_release == 'baselite' -%} -# baselite: All the axioms as they are editted by the editors, excluding reasoning. This is currently the same as "base". Only to be used experimentally (may disappear in future ODK releases). -$(ONT)-baselite.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) +{% if "baselite" in project.release_artefacts or project.primary_release == "baselite" -%} +# Baselite: All the axioms as intended by the editors (no imports, no reasoning). +{# FIXME: This is undocumented and of dubious usefulness. This was previously + said in a comment here to be "the same as base", which it is absolutely not. + See https://github.com/INCATools/ontology-development-kit/issues/1102. -#} +$(ONT)-baselite.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% if 'full' in project.release_artefacts or project.primary_release == 'full' -%} +{% if "full" in project.release_artefacts or project.primary_release == "full" -%} # Full: The full artefacts with imports merged, reasoned. -$(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) +$(ONT)-full.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} $(ROBOT_RELEASE_IMPORT_MODE) \ - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} \{% if project.release_materialize_object_properties is defined and project.release_materialize_object_properties %} - materialize {% for iri in project.release_materialize_object_properties %}--term {{iri}} {% endfor %} \{% endif %} - relax $(RELAX_OPTIONS) \ - reduce -r $(REASONER) $(REDUCE_OPTIONS) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + {% if project.release_materialize_object_properties -%} + materialize {# -#} + {% for iri in project.release_materialize_object_properties -%} + --term {{ iri -}} + {% if not loop.last %} \ + {# -#} + {% else %} \ + {% endif -%} + {% endfor -%} + {% endif -%} + relax $(RELAX_OPTIONS) \ + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% if 'non-classified' in project.release_artefacts or project.primary_release == 'non-classified' -%} -# foo-non-classified: (edit->imports-merged) -$(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) +{% if "non-classified" in project.release_artefacts or project.primary_release == "non-classified" -%} +# Non-classified: All the axioms as intended by the editors, +# including imported axioms but without reasoning. +$(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} $(ROBOT_RELEASE_IMPORT_MODE) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% if 'simple' in project.release_artefacts or project.primary_release == 'simple' -%} -# foo-simple: (edit->reason,relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest") -# drop every axiom: filter --term-file keep_terms.txt --trim true -# remove --select imports --trim false -$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) | all_robot_plugins - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% endif %} - relax $(RELAX_OPTIONS) \ - remove --axioms equivalent \ - filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \{% if project.release_use_reasoner %} - reduce -r $(REASONER) $(REDUCE_OPTIONS) \{% endif %} - odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true \ - repair --merge-axiom-annotations true \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ +{% if "simple" in project.release_artefacts or project.primary_release == "simple" -%} +# Simple: A version of the ontology that is {% if not project.release_use_reasoner %}un{% endif%}reasoned +# and from which all axioms referring to an entity outside +# of the ontology's namespace(s) are forcibly removed. +$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} | all_robot_plugins + $(ROBOT_RELEASE_IMPORT_MODE) \ + {% if project.release_use_reasoner -%} + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default("false")|lower }} \ + {% endif -%} + relax $(RELAX_OPTIONS) \ + remove --axioms equivalent \ + filter --term-file $(SIMPLESEED) \ + --select "annotations ontology anonymous self" \ + --trim true --signature true \ + {% if project.release_use_reasoner -%} + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + {% endif -%} + odk:normalize --base-iri {{ project.uribase }} \ + --subset-decls true --synonym-decls true \ + repair --merge-axiom-annotations true \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% if 'simple-non-classified' in project.release_artefacts or project.primary_release == 'simple-non-classified' %} -# foo-simple-non-classified (edit->relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest") - aka the HPO use case, no reason. -# Should this be the non-classified ontology with the drop foreign axiom filter? -# Consider adding remove --term "http://www.geneontology.org/formats/oboInOwl#hasOBONamespace" -$(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) +{% if "simple-non-classified" in project.release_artefacts or project.primary_release == "simple-non-classified" %} +# Simple-non-classified: Does not merge imports AND forcibly removes +# axioms outside of the ontology's namespaces(s). No reasoning. +$(ONT)-simple-non-classified.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} $(ROBOT_RELEASE_IMPORT_MODE_BASE) \ - remove --axioms equivalent \{% if project.release_use_reasoner %} - reduce -r $(REASONER) $(REDUCE_OPTIONS) \{% endif %} - filter --select ontology --term-file $(SIMPLESEED) --trim false \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ + remove --axioms equivalent \ + {% if project.release_use_reasoner -%} + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + {% endif -%} + filter --select ontology --term-file $(SIMPLESEED) --trim false \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% if 'international' in project.release_artefacts or project.primary_release == 'international' -%} -# international: A variant of the primary_release, but with multi-language support. -$(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) - $(ROBOT) merge $(patsubst %, -i %, $^) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \ - {% if project.release_date -%} --annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ +{% if "basic" in project.release_artefacts or project.primary_release == "basic" %} +# Basic: A version of the ontology that is {% if not project.release_use_reasoner %}un{% endif%}reasoned +# and from which all axioms referring to an entity outside +# of the ontology's namespace(s) are forcibly removed, as well +# as all relationships not whitelisted in KEEPRELATIONS. +$(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(SIMPLESEED) $(KEEPRELATIONS) + {%- if project.use_dosdps or project.components is not none %} $(OTHER_SRC){% endif %} + {%- if project.import_group is not none %} $(IMPORT_FILES){% endif %} + $(ROBOT_RELEASE_IMPORT_MODE) \ + {% if project.release_use_reasoner -%} + reason --reasoner $(REASONER) \ + --equivalent-classes-allowed {{ project.allow_equivalents }} \ + --exclude-tautologies {{ project.exclude_tautologies }} \ + --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default("false")|lower }} \ + {% endif -%} + relax $(RELAX_OPTIONS) \ + remove --axioms equivalent \ + remove --axioms disjoint \ + remove --term-file $(KEEPRELATIONS) \ + --select complement --select object-properties --trim true \ + filter --term-file $(SIMPLESEED) \ + --select "annotations ontology anonymous self" \ + --trim true --signature true \ + reduce --reasoner $(REASONER) $(REDUCE_OPTIONS) \ + $(SHARED_ROBOT_COMMANDS) \ + annotate --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% if 'basic' in project.release_artefacts or project.primary_release == 'basic' %} -# foo-basic: A version of -simple containing only relationships using relations on a configurable whitelist (default = BFO:0000050 (?)). -# See above (David comment) for explanation. -# removes any axioms that contains one of the ops that not in the whitelist file -$(ONT)-basic.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(KEEPRELATIONS) $(IMPORT_FILES) - $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} - reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% endif %} - relax $(RELAX_OPTIONS) \ - remove --axioms equivalent \ - remove --axioms disjoint \ - remove --term-file $(KEEPRELATIONS) --select complement --select object-properties --trim true \ - filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \ - reduce -r $(REASONER) $(REDUCE_OPTIONS) \ - $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ + +{% if "international" in project.release_artefacts or project.primary_release == "international" -%} +# International: A variant of the primary release, but with multi-language support. +$(ONT)-international.owl: $(ONT).owl $(TRANSLATIONS_OWL) + $(ROBOT) merge $(foreach src, $^, --input $(src)) \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + {% if project.release_date -%} + --annotation oboInOwl:date "$(OBODATE)" \ + {% endif -%} + --output $@ + {% endif -%} -{% for r in project.release_artefacts %} -{%- if r.startswith('custom-') %} + +{% for r in project.release_artefacts -%} +{% if r.startswith("custom-") -%} {{ r | replace("custom-","")}}.owl: - echo "ERROR: You have configured a custom release artefact ($@); this release artefact needs to be define in {{ project.id }}.Makefile!" && false -{% endif -%} + @echo "ERROR: You have configured a custom release artefact ($@);" + @echo " this artefact needs to be defined in {{ project.id }}.Makefile!" + @false + +{% endif -%} {% endfor -%} -# ---------------------------------------- -# Debugging Tools +{% if project.export_formats|length > 1 or project.export_formats[0] != "owl" -%} +# Release artefacts: export formats # ---------------------------------------- -explain_unsat: $(EDIT_PREPROCESSED) - $(ROBOT) explain -i $< -M unsatisfiability --unsatisfiable random:10 --explanation $(TMPDIR)/$@.md +{# FIXME: The seeding script should provide us with a list of artefacts + that include both the "primary" artefact ($(ONT)) plus all the + variants ($(ONT)-base, $(ONT)-full, etc.), so that we do not have + to duplicate all the code below (once for the primary artefact and + one when we iterate over the variants. -#} +{% if "obo" in project.export_formats -%} +$(ONT).obo: $(ONT).owl + $(ROBOT) convert --input $< --check false \ + --format obo $(OBO_FORMAT_OPTIONS) --output $@ -{% if project.public_release == 'github' -%} +{% endif -%} +{% if "ttl" in project.export_formats -%} +{# FIXME: Why are re-annotating? The release.owl product should have + been annotated already. + See https://github.com/INCATools/ontology-development-kit/issues/1366. -#} +$(ONT).ttl: $(ONT).owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format ttl --output $@ + +{% endif -%} +{% if "json" in project.export_formats -%} +$(ONT).json: $(ONT).owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format json --output $@ + +{% endif -%} +{% for r in project.release_artefacts -%} +{% if r.startswith("custom-") -%} +{% set release = r | replace("custom-","") -%} +{% else -%} +{% set release = "$(ONT)-" ~ r -%} +{% endif -%} +{% if "obo" in project.export_formats -%} +{{ release }}.obo: {{ release }}.owl + $(ROBOT) convert --input $< --check false \ + --format obo $(OBO_FORMAT_OPTIONS) --output $@ + +{% endif -%} +{% if "ttl" in project.export_formats -%} +{{ release }}.ttl: {{ release }}.owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format ttl --output $@ + +{% endif -%} +{% if "json" in project.export_formats -%} +{{ release }}.json: {{ release }}.owl + $(ROBOT) annotate --input $< --ontology-iri $(ONTBASE)/$@ \ + $(ANNOTATE_ONTOLOGY_VERSION) \ + convert --check false --format json --output $@ + +{% endif -%} +{% endfor -%} + +{% if 'db' in project.export_formats -%} + +{% if project.use_context -%} +CONTEXT_FILE_CSV = $(TMPDIR)/context.csv +$(CONTEXT_FILE_CSV): $(CONTEXT_FILE) | $(TMPDIR) + @odk-helper context2csv $< > $@ + +{% endif -%} + +%.db: %.owl{% if project.use_context %} $(CONTEXT_FILE_CSV){% endif %} + @rm -f $*.db $*-relation-graph.tsv.gz .template.db .template.db.tmp + semsql make $*.db{% if project.use_context %} -P $(CONTEXT_FILE_CSV){% endif %} + @rm -f $*-relation-graph.tsv.gz .template.db .template.db.tmp + @test -f $*.db || (echo "SQLite/SemSQL generation failed" && exit 1) + +{% endif -%} + +{% for format in project.export_formats -%} +{% if project.gzip_main -%} +$(ONT).{{ format }}.gz: $(ONT).{{ format }} + gzip -c $< > $@ + +{% endif -%} +{% endfor -%} +{% if 'owl' not in project.export_formats -%} +{% if project.gzip_main -%} +$(ONT).owl.gz: $(ONT).owl + gzip -c $< > $@ + +{% endif -%} +$(ONT).owl: $(ONT)-{{ project.primary_release }}.owl + cp $< $@ + +{% endif -%} + +{% endif %}{# !if project.export_formats|length > 1 or project.export_formats[0] != "owl" -#} + +{# #} +# ---------------------------------------- +# Release management +# ---------------------------------------- + +# Assets definitions and helper targets +# ---------------------------------------- + +ASSETS = $(MAIN_FILES){# -#} + {% if project.import_group is not none %} \ + $(IMPORT_FILES){# -#} + {% endif -%} + {% if project.orcidio_support %} \ + $(IMPORTDIR)/orcidio_import.owl{# -#} + {% endif -%} + {% if project.use_dosdps %} \ + $(PATTERN_RELEASE_FILES){# -#} + {% endif -%} + {% if project.use_translations %} \ + $(TRANSLATION_FILES){# -#} + {% endif -%} + {% if project.subset_group is not none %} \ + $(SUBSET_FILES){# -#} + {% endif -%} + {% if project.use_mappings %} \ + $(MAPPING_FILES){# -#} + {% endif -%} + {% if project.bridge_group is not none %} \ + $(BRIDGE_FILES){# -#} + {% endif %} \ + $(REPORT_FILES) + +# All the files that are part of a release (excepted mappings). +{# This does not include mappings (which can optionally be released), + because those are stored in src/mappings, and the fact that this is + one directory _above_ the directory where this Makefile is would + cause problems down the road (notably, it would prevent the use of a + single rsync commmand to copy all release assets except mappings to + the release directory). Basically, this is "all release assets that + are in this src/ontology directory". -#} +RELEASE_ASSETS = $(MAIN_FILES){# -#} + {% if project.import_group.release_imports %} \ + $(IMPORT_FILES){# -#} + {% endif -%} + {% if project.subset_group is not none %} \ + $(SUBSET_FILES){# -#} + {% endif -%} + {% if project.bridge_group is not none %} \ + $(BRIDGE_FILES){# -#} + {% endif -%} + {% if project.robot.report.release_reports %} \ + $(REPORT_FILES){# -#} + {% endif %} + +# All the files that are part of a release, in their final location +# within the released directory. +{# This must be kept in sync with what the copy_release_files rule below + does -- that is, this should list all files copied by that rule. -#} +RELEASED_ASSETS = $(foreach n, $(RELEASE_ASSETS), $(RELEASEDIR)/$(n)){# -#} + {% if project.sssom_mappingset_group.released_products is defined %} \ + $(foreach n, $(RELEASED_MAPPINGS), $(RELEASEDIR)/mappings/$(n).sssom.tsv){# -#} + {% endif %} + +# The files that must be removed from the src/ontology directory +# once they have been copied over to the release directory. +CLEANFILES = $(MAIN_FILES) + +.PHONY: all_assets +all_assets: $(ASSETS) + +.PHONY: show_assets +show_assets: + @echo $(ASSETS) + @du -sh $(ASSETS) + +# Release preparation # ---------------------------------------- -# GitHub release + +# This is the main release workflow that should be executed by the release +# manager whenever the time comes to make a release. It will ensure that +# all assets are fresh, and copy them over to the release directory. +.PHONY: prepare_release +prepare_release: all_odk + $(MAKE) copy_release_files + rm -f $(CLEANFILES) + @echo "INFO: Release files are now in $(RELEASEDIR); now you should commit," + @echo " push and make release on your Git hosting site." + +.PHONY: prepare_release_fast +prepare_release_fast: + $(MAKE_FAST) prepare_release + +{# This rule does the bulk of the work for prepare_release, copying all + files to the release directory. It is mostly intended for internal + use, by the prepare_release rule itself or by other ODK scripts. -#} +.PHONY: copy_release_files +copy_release_files: + rsync -R $(RELEASE_ASSETS) $(RELEASEDIR){# -#} + {% if project.sssom_mappingset_group.released_products is defined %} + mkdir -p $(RELEASEDIR)/mappings + cp -rf $(RELEASED_MAPPING_FILES) $(RELEASEDIR)/mappings{# -#} + {% endif %} + +.PHONY: show_release_assets +show_release_assets: + @echo $(RELEASED_ASSETS) + +# Publication # ---------------------------------------- +{% if project.public_release == 'github' -%} GHVERSION = v$(VERSION) .PHONY: public_release public_release: - @echo "Pushing public release $(GHVERSION)..." - @ls -alt $(RELEASE_ASSETS_AFTER_RELEASE) - @gh release create $(GHVERSION) \ - --title "$(VERSION) Release" \ - --generate-notes \ - $(RELEASE_ASSETS_AFTER_RELEASE) + @echo "Pushing public release $(GHVERSION) to GitHub..." + @echo "Assets to be pushed:" + @ls -alt $(RELEASED_ASSETS) + @gh release create $(GHVERSION) + --title "$(VERSION) Release" \ + --generate-notes \ + $(RELEASED_ASSETS) {% else -%} -# ---------------------------------------- -# Public release -# ---------------------------------------- .PHONY: public_release public_release: @@ -1565,24 +1991,10 @@ public_release: @false {% endif -%} +{# #} # ---------------------------------------- -# General Validation -# ---------------------------------------- -TSV= -ALL_TSV_FILES={% if project.use_dosdps %}{% if project.pattern_pipelines_group is defined -%}{% for pipeline in project.pattern_pipelines_group.products %}$(DOSDP_TSV_FILES_{{ pipeline.id | upper }}) {% endfor %}{% endif %}$(DOSDP_TSV_FILES_DEFAULT){% endif %} - -validate-tsv: $(TSV) | $(TMPDIR) - for FILE in $< ; do \ - tsvalid $$FILE > $(TMPDIR)/validate.txt; \ - if [ -s $(TMPDIR)/validate.txt ]; then cat $(TMPDIR)/validate.txt && exit 1; fi ; \ - done - -validate-all-tsv: $(ALL_TSV_FILES) - $(MAKE) validate-tsv TSV="$^" - -# ---------------------------------------- -# Editors Utilities +# Utilities # ---------------------------------------- # This is an experimental target people that want to use ODK Extended Prefix Map (EPM) @@ -1592,113 +2004,208 @@ validate-all-tsv: $(ALL_TSV_FILES) $(EXTENDED_PREFIX_MAP): $(ODK_RESOURCES_DIR)/obo.epm.json cp $< $@ -{% if 'obo' in project.edit_format -%} -.PHONY: normalize_obo_src -normalize_obo_src: $(SRC) | all_robot_plugins - $(ROBOT) repair -i $< --merge-axiom-annotations true \ - convert -o $(TMPDIR)/NORM.tmp.obo && \ - mv $(TMPDIR)/NORM.tmp.obo $(SRC) -{%- endif %} - -.PHONY: normalize_src -normalize_src: $(SRC) - $(ROBOT) convert -i $< -f {% if 'obo' == project.edit_format %}obo --check false{% else %}ofn{% endif %} -o $(TMPDIR)/normalise && mv $(TMPDIR)/normalise $< +# Repository maintenance +# ---------------------------------------- .PHONY: validate_idranges validate_idranges: if [ -f {{ project.id }}-idranges.owl ]; then \ - dicer-cli policy --assume-manchester --show-owlapi-error {{ project.id }}-idranges.owl ; \ + dicer-cli policy --assume-manchester --show-owlapi-error {{ project.id }}-idranges.owl ; \ fi -# Deprecated: Use 'sh run.sh odk.py update' without using the Makefile. +# Deprecated: Use 'sh run.sh odk update' without using the Makefile. .PHONY: update_repo update_repo: - odk.py update - -{% if project.documentation is not none %} + odk update + +{% if project.documentation is not none -%} +.PHONY: update_docs: mkdocs gh-deploy --config-file ../../mkdocs.yaml -{%- endif %} -# Note to future generations: computing the real path relative to the -# current directory is a way to ensure we only clean up directories that -# are located below the current directory, regardless of the contents of -# the *DIR variables. +{% endif -%} + +{# Note to future generations: computing the real path relative to the + current directory is a way to ensure we only clean up directories + that are located below the current directory, regardless of the + contents of the *DIR variables. -#} .PHONY: clean -clean:{% if project.use_dosdps %} - $(MAKE) pattern_clean{%- endif %} - for dir in $(MIRRORDIR) $(TMPDIR) $(UPDATEREPODIR) ; do \ - reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \ - case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \ +clean: + {% if project.use_dosdps -%} + $(MAKE) pattern_clean + {% endif -%} + for dir in $(MIRRORDIR) $(TMPDIR) ; do \ + reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \ + case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \ done rm -f $(CLEANFILES) - rm -f $(TMPDIR)/download-*.info + +{% if "obo" == project.edit_format -%} +.PHONY: normalize_obo_src +normalize_obo_src: $(SRC) | all_robot_plugins + $(ROBOT) repair --input $< --merge-axiom-annotations true \ + convert --output $(TMPDIR)/normalized-$< \ + && mv $(TMPDIR)/normalized-$< $< + +{% endif -%} + +.PHONY: normalize_src +normalize_src: $(SRC) + $(ROBOT) convert --input $< --format {# -#} + {% if "obo" == project.edit_format -%} + obo --check false {# -#} + {% else -%} + ofn {# -#} + {% endif -%} + --output $(TMPDIR)/normalized-$< \ + && mv $(TMPDIR)/normalized-$< $< + +# Debugging tools +# ---------------------------------------- + +.PHONY: explain_unsat +explain_unsat: $(EDIT_PREPROCESSED) + $(ROBOT) explain --input $< \ + --mode unsatisfiability --unsatisfiable random:10 \ + --explanation $(TMPDIR)/$@.md + +# Validation of TSV files +# ---------------------------------------- + +{% if project.use_dosdps -%} +ALL_TSV_FILES = $(DOSDP_TSV_FILES_DEFAULT){# -#} + {% if project.pattern_pipelines_group is not none -%} + {% for pipeline in project.pattern_pipelines_group.products %} \ + $(DOSDP_TSV_FILES_{{ pipeline.id | upper }}){# -#} + {% endfor -%} + {% endif %} + +.PHONY: validate-all-tsv +validate-all-tsv: $(ALL_TSV_FILES) + $(MAKE) validate-tsv TSV="$^" + +{% endif -%} + +{# FIXME: This is of dubious usefulness; if the user needs to validate + an arbitrary TSV file, she might as well run "tsvalid the-file.tsv". -#} +.PHONY: validate-tsv +validate-tsv: $(TSV) | $(TMPDIR) + for FILE in $< ; do \ + tsvalid $$FILE > $(TMPDIR)/validate-tsv.txt ; \ + [ ! -s $(TMPDIR)/validate-tsv.txt ] || (cat $(TMPDIR)/validate-tsv ; false) ; \ + done + +# "Release diff" +# ---------------------------------------- + +# This produces a diff between the last local build of the ontology +# and the version currently available online, +{# FIXME: This is quite OBO-specific. -#} +CURRENT_RELEASE = $(ONTBASE).owl + +$(TMPDIR)/current-release.owl: + wget $(CURRENT_RELEASE) --output-document $@ + +$(REPORTDIR)/release-diff.md: $(ONT).owl $(TMPDIR)/current-release.owl + $(ROBOT) diff --left $(TMPDIR)/current-release.owl \ + --right $(ONT).owl \ + --labels true --format markdown --output $@ + + +# ---------------------------------------- +# Online help +# ---------------------------------------- .PHONY: help help: - @echo "$$data" + @echo "$$ODK_HELP_MESSAGE" + +define ODK_HELP_MESSAGE +Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(MIR|IMP|IMP_LARGE|COMP|PAT)=(false|true)] command -define data -Usage: [IMAGE=(odklite|odkfull)] [ODK_DEBUG=yes] sh run.sh make [(IMP|MIR|IMP_LARGE|PAT)=(false|true)] command + or: odkrun [--lite] [--debug] make [(MIR|IMP|IMP_LARGE|COMP|PAT)=(false|true)] command ---------------------------------------- Command reference ---------------------------------------- Core commands: -* prepare_release: Run the entire release pipeline. Use make IMP=false prepare_release to avoid rerunning the imports -* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns. -* update_repo: Update the ODK repository setup using the config file {{ project.id }}-odk.yaml (DEPRECATED) -* test: Running all validation tests -* test_fast: Runs the test suite, but without updating imports or components +* prepare_release: Run the entire release pipeline. Use make MIR=false prepare_release to + avoid refreshing remote resources. +* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating + components or recompiling patterns. +* test: Running all validation tests. +* test_fast: Runs the test suite, but without updating imports or components. * odkversion: Show the current version of the ODK Makefile and ROBOT. -* clean: Delete all temporary files -* help: Print ODK Usage information -* public_release: Uploads the release file to a release management system, such as GitHub releases. Must be configured. +* clean: Delete all temporary files. +* help: Print ODK Usage information. +* public_release: Uploads the release file to a release management system, such as GitHub + releases. Must be configured. -{% if project.import_group is defined %} +{% if project.import_group is not none -%} Imports management: * refresh-imports: Refresh all imports and mirrors. * recreate-components: Recreate all components. * no-mirror-refresh-imports: Refresh all imports without downloading mirrors. -* refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled as 'is_large'. -* refresh-%: Refresh a single import, i.e. refresh-go will refresh 'imports/go_import.owl'. -* no-mirror-refresh-%: Refresh a single import without updating the mirror, i.e. refresh-go will refresh 'imports/go_import.owl'. +* refresh-imports-excluding-large: Refresh all imports and mirrors, but skipping the ones labelled + as 'is_large'. +{% if not project.import_group.use_base_merging -%} +* refresh-%: Refresh a single import + (i.e. refresh-go will refresh 'imports/go_import.owl'). +* no-mirror-refresh-%: Refresh a single import without updating the mirror + (i.e. refresh-go will refresh 'imports/go_import.owl'). +{% endif -%} * mirror-%: Refresh a single mirror. -{% endif %}{% if project.use_dosdps %} + +{% endif -%} +{% if project.use_dosdps -%} DOSDP templates -* dosdp: Run the DOSDP patterns pipeline: Run tests, then build OWL files from the tables. -* patterns: Alias of the 'dosdp' command -* pattern_clean: Delete all temporary pattern files -* dosdp_validation: Run all validation checks on DOSDP template files and tables -* pattern_schema_checks: Alias of the 'dosdp_validation' command -* update_patterns: Pull updated patterns listed in dosdp-patterns/external.txt -* dosdp-matches-%: Run the DOSDP matches/query pipeline as configured in your {{ project.id }}-odk.yaml file. +* dosdp: Run the DOSDP patterns pipeline: Run tests, then build OWL files + from the tables. +* patterns: Alias of the 'dosdp' command. +* pattern_clean: Delete all temporary pattern files. +* dosdp_validation: Run all validation checks on DOSDP template files and tables. +* pattern_schema_checks: Alias of the 'dosdp_validation' command. +* update_patterns: Pull updated patterns listed in dosdp-patterns/external.txt. +* dosdp-matches-%: Run the DOSDP matches/query pipeline as configured in your project. * dosdp-docs-%: Generate the documentation for a given DOSDP pipeline. -{% endif %} + +{% endif -%} Editor utilities: -* validate_idranges: Make sure your ID ranges file is formatted correctly -* normalize_src: Load and save your {{ project.id }}-edit file after you to make sure its serialised correctly{% if 'obo' in project.edit_format %} -* normalize_obo_src: Load and save your {{ project.id }}-edit.obo file after you to merge duplicate annotation assertions{% endif %} -* explain_unsat: If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes -* validate-all-tsv: Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files -* validate-tsv: Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv. -* release_diff: Create a diff between the current release and the new release - -Additional build commands (advanced users) -* all: Run the entire pipeline (like prepare_release), but without copying the release files to the release directory. -* all_subsets: Build all subsets -* custom_reports: Generate all custom sparql reports you have configured in your {{ project.id }}-odk.yaml file. -* all_assets: Build all assets -* show_assets: Print a list of all assets that would be build by the release pipeline -* all_mappings: Update all SSSOM mapping sets - -Additional QC commands (advanced users) -* robot_reports: Run all configured ROBOT reports -* validate_profile_%: Run an OWL2 DL profile validation check, for example validate_profile_{{ project.id }}-edit.owl. -* reason_test: Run a basic reasoning test - -Examples: +* validate_idranges: Make sure your ID ranges file is formatted correctly. +* normalize_src: Load and save your {{ project.id }}-edit file after you to make sure it is + serialised correctly +{% if "obo" == project.edit_format -%} +* normalize_obo_src: Load and save your {{ project.id }}-edit.obo file after you to merge + duplicate annotation assertions. +{% endif -%} +* explain_unsat: Produce explanations (in $(TMPDIR)/explain_unsat.md) for a random sample of + unsatisfiable classes in your ontology (if any). +{% if project.use_dosdps -%} +* validate-all-tsv: Check all your tsv files for possible problems in syntax. +{% endif -%} +* release_diff: Create a diff between the current release and the new release + +Additional build commands (advanced users): +* all: Run the entire pipeline (like prepare_release), but without copying the + release files to the release directory. +{% if project.subset_group is not none -%} +* all_subsets: Build all subsets. +{% endif -%} +{% if project.use_mappings -%} +* all_mappings: Update all SSSOM mapping sets. +{% endif -%} +* custom_reports: Generate all custom sparql reports you have configured in your project. +* all_assets: Build all assets. +* show_assets: Print a list of all assets that would be build by the release pipeline. + +Additional QC commands (advanced users): +* robot_reports: Run all configured ROBOT reports. +* validate_profile_%: Run an OWL2 DL profile validation check on an arbitrary file + (i.e. validate_profile_{{ project.id }}-edit.owl). +* reason_test: Run a basic reasoning test. + +Examples: * sh run.sh make IMP=false prepare_release * sh run.sh make test @@ -1707,13 +2214,16 @@ Tricks: invoke the 'clean' target (which will wipe out intermediate files) or touch a file that your target depends on (typically the -edit file). * Use the IMAGE parameter to the run.sh script to use a different image like odklite + (or use odkrun's --image option). * Use ODK_DEBUG=yes sh run.sh make ... to print information about timing and debugging + (or use odkrun's --debug option). Updating the repository: (to apply changes to the ODK configuration or switch to a newer ODK version) * sh run.sh update_repo + (or odkrun odk update) endef -export data +export ODK_HELP_MESSAGE include {{ project.id }}.Makefile