From 93b3e9e8d207857dccb87e4330451836e17f4f18 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 22 Apr 2026 11:07:12 -0500 Subject: [PATCH 001/222] add methods and adjust row handling to accommodate subset regex --- lib/datura/file_types/file_csv.rb | 43 ++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index c3336cb54..a2bebbb5f 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -64,16 +64,22 @@ def transform_es puts "transforming #{self.filename}" es_doc = [] + row_filter = build_csv_row_filter + if row_filter + puts "csv_rows filter active: only processing rows matching /#{@options["csv_rows"]}/".cyan + end + @csv.each do |row| - if !row.header_row? - row_to_es = row_to_es(@csv.headers, row) - if !row_to_es["identifier"].to_s.empty? && !row_to_es["title"].to_s.empty? - es_doc << row_to_es - else - puts "skipping item without id or title".red - puts "check line ".red + row.to_s.strip[0..400].red - next - end + next if row.header_row? + next if row_filter && !row_matches_filter?(row, row_filter) + + row_to_es = row_to_es(@csv.headers, row) + if !row_to_es["identifier"].to_s.empty? && !row_to_es["title"].to_s.empty? + es_doc << row_to_es + else + puts "skipping item without id or title".red + puts "check line ".red + row.to_s.strip[0..400].red + next end end if @options["output"] @@ -123,4 +129,23 @@ def write_html_to_file(builder, index) puts "writing to #{filepath}" if @options["verbose"] File.open(filepath, "w") { |f| f.write(builder.to_xml) } end + + private + + def build_csv_row_filter + return nil unless @options["csv_rows"] + + begin + Regexp.new(@options["csv_rows"]) + rescue RegexpError => e + puts "Warning: --csv-rows value '#{@options["csv_rows"]}' is not a valid regex: #{e.message}".red + puts "Proceeding without row filter — all rows will be processed.".yellow + nil + end + end + + def row_matches_filter?(row, filter) + id = row["id"] || row["identifier"] || row["Identifier"] || "" + !!filter.match(id) + end end From 278a0d856223d83a181082ed94e0d2f6813ae714 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 22 Apr 2026 11:08:43 -0500 Subject: [PATCH 002/222] add c option for csv subset regex --- lib/datura/parser_options/post.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index b6e48880f..3b32ba689 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -49,6 +49,13 @@ def self.post_params options["regex"] = input end + options["csv_rows"] = nil + opts.on('-c', '--csv-rows INPUT', + 'Only process CSV rows whose identifier (id/identifier column) matches this regex.', + 'Examples: --csv-rows nca_001 --csv-rows ^nca_00[1-3] --csv-rows nca_001|nca_007') do |input| + options["csv_rows"] = input + end + options["transform_only"] = false opts.on('-t', '--transform-only', 'Do not post to solr / es') do options["transform_only"] = true From 5d15c5977c8b9ec2e8e5a3f69f1e15aa9b968ec9 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 22 Apr 2026 11:13:48 -0500 Subject: [PATCH 003/222] adjust help syntax --- lib/datura/parser_options/post.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index 3b32ba689..16522eec3 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -50,7 +50,7 @@ def self.post_params end options["csv_rows"] = nil - opts.on('-c', '--csv-rows INPUT', + opts.on('-c', '--csv-rows [input]', 'Only process CSV rows whose identifier (id/identifier column) matches this regex.', 'Examples: --csv-rows nca_001 --csv-rows ^nca_00[1-3] --csv-rows nca_001|nca_007') do |input| options["csv_rows"] = input From 5d7d182027d8b6c49f0483ac880efed87d4faa63 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 22 Apr 2026 12:00:10 -0500 Subject: [PATCH 004/222] remove examples to avoid confusion --- lib/datura/parser_options/post.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index 16522eec3..511eb03b2 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -51,8 +51,7 @@ def self.post_params options["csv_rows"] = nil opts.on('-c', '--csv-rows [input]', - 'Only process CSV rows whose identifier (id/identifier column) matches this regex.', - 'Examples: --csv-rows nca_001 --csv-rows ^nca_00[1-3] --csv-rows nca_001|nca_007') do |input| + 'Only process CSV rows whose identifier (id/identifier column) matches this regex.') do |input| options["csv_rows"] = input end From 328d1673c740cc50ab3af3a3845d6cb828ae1af0 Mon Sep 17 00:00:00 2001 From: Greg Tunink Date: Wed, 22 Apr 2026 17:43:09 -0500 Subject: [PATCH 005/222] Copy changelog template for upcoming changes --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b12b25c3d..0973b094f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,23 @@ Versioning](https://semver.org/spec/v2.0.0.html). ### Security --> +## [Unreleased] - Brief description TBD before next release +[Unreleased]: https://github.com/CDRH/datura/compare/v1.1.0...dev + +### Fixed + +### Added + +### Changed + +### Removed + +### Migration + +### Deprecated + +### Security + ## [v1.1.0] - Omeka S Posting [v1.1.0]: https://github.com/CDRH/datura/compare/v1.0.1...v1.1.0 From 316aeb4a11f43065bca562077283920432b36fda Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 29 Apr 2026 13:34:30 -0500 Subject: [PATCH 006/222] add backtrace for verbose option, shift to message method --- lib/datura/file_type.rb | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 2e53fd174..a68139a1a 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -35,6 +35,12 @@ def initialize(location, options) # script locations set in child classes end + def debug_info(error) + if options["verbose"] + puts "Backtrace: " << error.backtrace.join("\n ") + end + end + def filename(ext=true) if ext File.basename(@file_location) @@ -56,6 +62,7 @@ def post_es(es) begin transformed = transform_es rescue => e + debug_info(e) return { "error" => "Error transforming ES for #{self.filename(false)}: #{e.full_message}" } end if transformed && transformed.length > 0 @@ -76,7 +83,9 @@ def post_es(es) begin RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) ) rescue => e - error = "Error transforming or posting to ES for #{self.filename(false)}: #{e}" + debug_info(e) + puts "Erroneous document ID: #{id}" + error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.message}" end else error = "Document #{id} did not validate against the elasticsearch schema" @@ -109,7 +118,8 @@ def post_solr(url=nil) return { "error" => "Error posting to Solr for #{self.filename}: #{res.body}" } end rescue => e - return { "error" => "Error posting to Solr for #{self.filename}: #{e.inspect}" } + debug_info(e) + return { "error" => "Error posting to Solr for #{self.filename}: #{e.message}" } end end @@ -147,7 +157,7 @@ def transform_es end return es_req rescue => e - puts "something went wrong transforming #{self.filename}" + puts "something went wrong transforming #{self.filename}: #{e.message}" puts e puts e.backtrace raise e From 5438062386bd3240022d894e33ff17ee763c7b5e Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 29 Apr 2026 13:40:04 -0500 Subject: [PATCH 007/222] remove redundant id note --- lib/datura/file_type.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index a68139a1a..b1cccc8f7 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -84,7 +84,6 @@ def post_es(es) RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) ) rescue => e debug_info(e) - puts "Erroneous document ID: #{id}" error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.message}" end else From 6aacddd7a43d457ee759a5fe4dfc8bc94bd408dc Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 29 Apr 2026 15:12:02 -0500 Subject: [PATCH 008/222] add rescue block to post and prepare_xslt method, raise directory error --- bin/post | 9 +++++++-- lib/datura/data_manager.rb | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/post b/bin/post index 026b183f8..b0ec665be 100755 --- a/bin/post +++ b/bin/post @@ -2,5 +2,10 @@ require "datura" -manager = Datura::DataManager.new -manager.run +begin + manager = Datura::DataManager.new + manager.run +rescue => e + puts e.message.red + exit 1 +end \ No newline at end of file diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 24f4898e9..377757079 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -264,7 +264,13 @@ def prepare_xslt if !t1 || !t2 || t1 > t2 puts "Copying datura XSLT default scripts into collection" - FileUtils.cp_r(datura_xslt, dest) + begin + FileUtils.cp_r(datura_xslt, dest) + rescue Errno::ENOENT => e + raise "Could not copy XSLT scripts into the collection. " \ + "Confirm you are running this command from the root of the collection repository, " \ + "not from a subdirectory." + end end end From e018533c67dd6e371404f730764d5dffcc894050 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 29 Apr 2026 15:23:31 -0500 Subject: [PATCH 009/222] shorten Saxon error messages --- lib/datura/file_type.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index b1cccc8f7..5a3694147 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -209,8 +209,12 @@ def exec_xsl(input, xsl, ext, outpath=nil, params=nil) out = stdout.read err = stderr.read if err.length > 0 - msg = "There was an error transforming #{filename}: #{err}" - return { "error" => msg } + # Extract meaningful lines from Saxon's stderr for user display + key_lines = err.lines.select { |l| l.match?(/^Error|SXXP|XPST|XTTE|XTSE|XSLT|Fatal/) } + key_lines = err.lines.first(1) if key_lines.empty? + msg = "Error transforming #{filename}: \n " + key_lines.map{ |l| l.strip + "..."}.join("\n ") + msg += "\n (full Saxon output logged)" if err.lines.length > key_lines.length + return { "error" => msg, "full_error" => err } else puts "Successfully transformed #{filename}" return { "doc" => out } From e140a155005e751f252817726afb684ab14ff1b2 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 29 Apr 2026 15:36:43 -0500 Subject: [PATCH 010/222] print all errors at the end of the post --- lib/datura/data_manager.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 377757079..478dd49e0 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -160,6 +160,22 @@ def end_run puts error_msg @log.info(error_msg) + all_errors = { + "ES" => @error_es, + "HTML" => @error_html, + "IIIF" => @error_iiif, + "Solr" => @error_solr + }.reject { |_, v| v.empty? } + + if all_errors.any? + puts "\n--- Error details ---".red + all_errors.each do |type, errors| + errors.each { |e| puts "[#{type}] #{e}".red } + end + puts "---------------------".red + @log.error("Error details: #{all_errors.inspect}") + end + # figure time for running @time << Time.now dur = @time[1] - @time[0] From e058d813fd33bbf2348d82bc7f617deeee6c7fef Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 09:20:35 -0500 Subject: [PATCH 011/222] add rescue blocks for set_up_services and transform_and_post --- lib/datura/data_manager.rb | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 478dd49e0..9e2d72355 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -211,7 +211,7 @@ def options_msg msg << "Running script with following options:\n" msg << "collection: #{@options['collection']}\n" msg << "Environment: #{@options['environment']}\n" - msg << "Posting to: #{@es.index_url}\n\n" if should_post?("es") + msg << "Posting to: #{@es.index_url}\n\n" if should_post?("es") && @es msg << "Posting to: #{@solr_url}\n\n" if should_post?("solr") msg << "Format: #{@options['format']}\n" if @options["format"] msg << "Regex: #{@options['regex']}\n" if @options["regex"] @@ -305,8 +305,16 @@ def set_up_logger def set_up_services if should_post?("es") - # set up elasticsearch instance - @es = Datura::Elasticsearch::Index.new(@options, schema_mapping: true) + begin + # set up elasticsearch instance + @es = Datura::Elasticsearch::Index.new(@options, schema_mapping: true) + rescue Errno::ECONNREFUSED, SocketError, Errno::ETIMEDOUT => e + msg = "Could not connect to Elasticsearch at #{File.join(@options['es_path'], @options['es_index'])}. " \ + "Confirm you have specified the correct environment " \ + "(currently: #{@options['environment']}). Use -e to specify an environment." + error_with_transform_and_post(msg, @error_es) + @es = nil + end end if should_post?("solr") @@ -326,21 +334,19 @@ def should_transform?(type) def transform_and_post(file) # elasticsearch - if should_transform?("es") - if @options["transform_only"] - # TODO transformation is not treated the same way here as in - # most post methods, so having to use try catch block - begin + begin + if should_transform?("es") + if @options["transform_only"] res_es = file.transform_es - rescue => e - error_with_transform_and_post("#{e}", @error_es) - end - else - res_es = file.post_es(@es) - if res_es && res_es.has_key?("error") - error_with_transform_and_post(res_es["error"], @error_es) + elsif @es + res_es = file.post_es(@es) + if res_es && res_es.has_key?("error") + error_with_transform_and_post(res_es["error"], @error_es) + end end end + rescue => e + error_with_transform_and_post("#{e}", @error_es) end # html From 681081776fbe84a9e2d9e2659a7bdb6483d9cb7e Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 09:47:33 -0500 Subject: [PATCH 012/222] add rescue specific to connection error, prompt user to check environment --- lib/datura/file_type.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 5a3694147..405e93f86 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -82,6 +82,10 @@ def post_es(es) # you will need to add _update at the end of this URL begin RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) ) + rescue Errno::ECONNREFUSED, SocketError, Errno::ETIMEDOUT => e + error = "Could not connect to ElasticSearch at #{es.index_url}. " \ + "Confirm you have specified the correct environment " \ + "(currently: #{@options['environment']}. Use -e to specify an environment." rescue => e debug_info(e) error = "Error transforming or posting to ES for #{self.filename(false)}: #{e.message}" From d045a599035203ae8fb60c40e809a971e31b3ba9 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 10:38:45 -0500 Subject: [PATCH 013/222] remove redundant error printing --- lib/datura/file_type.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 405e93f86..88fa53a42 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -160,9 +160,6 @@ def transform_es end return es_req rescue => e - puts "something went wrong transforming #{self.filename}: #{e.message}" - puts e - puts e.backtrace raise e end end From edcd33b5405838cceaee433789a4f71b91ed891f Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 10:40:41 -0500 Subject: [PATCH 014/222] adjust saxon error handling to display two lines per error --- lib/datura/file_type.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 88fa53a42..bd1157488 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -210,10 +210,13 @@ def exec_xsl(input, xsl, ext, outpath=nil, params=nil) out = stdout.read err = stderr.read if err.length > 0 - # Extract meaningful lines from Saxon's stderr for user display - key_lines = err.lines.select { |l| l.match?(/^Error|SXXP|XPST|XTTE|XTSE|XSLT|Fatal/) } - key_lines = err.lines.first(1) if key_lines.empty? - msg = "Error transforming #{filename}: \n " + key_lines.map{ |l| l.strip + "..."}.join("\n ") + # Extract meaningful lines from Saxon's stderr for user display + lines = err.lines + match_indices = lines.each_index.select { |i| lines[i].match?(/^Error|SXXP|XPST|XTTE|XTSE|XSLT|Fatal/) } + wanted_indices = match_indices.flat_map { |i| [i, i + 1] }.uniq.select { |i| i < lines.length } + key_lines = wanted_indices.map { |i| lines[i] } + key_lines = lines.first(2) if key_lines.empty? + msg = "Error transforming #{filename}: {\n " + key_lines.map { |l| l.strip }.join("\n ") msg += "\n (full Saxon output logged)" if err.lines.length > key_lines.length return { "error" => msg, "full_error" => err } else @@ -236,4 +239,4 @@ def subdoc_xpaths # } end -end +end \ No newline at end of file From 49c676203d028e560124ce747bdeea363c00c5ed Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 15:40:54 -0500 Subject: [PATCH 015/222] add non-normalized column names to broaden applicability of csv transformation --- lib/datura/to_es/csv_to_es.rb | 2 +- lib/datura/to_es/csv_to_es/fields.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/to_es/csv_to_es.rb b/lib/datura/to_es/csv_to_es.rb index 5625c6e84..9ca504084 100644 --- a/lib/datura/to_es/csv_to_es.rb +++ b/lib/datura/to_es/csv_to_es.rb @@ -41,7 +41,7 @@ def create_json end def get_id - @row["id"] || @row["identifier"] || "" + @row["id"] || @row["identifier"] || @row["Identifier"] || "" end def preprocessing diff --git a/lib/datura/to_es/csv_to_es/fields.rb b/lib/datura/to_es/csv_to_es/fields.rb index 6adcfc23c..b447b6762 100644 --- a/lib/datura/to_es/csv_to_es/fields.rb +++ b/lib/datura/to_es/csv_to_es/fields.rb @@ -196,7 +196,7 @@ def text_additional end def title - @row["title"] + @row["title"] || @row["Title"] end def title_sort From 951baab7c88b2aabf3407290119d467b90762322 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 16:44:11 -0500 Subject: [PATCH 016/222] add saxonche to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9cb662327..ab8ae4893 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ platformdirs==4.4.0 python-dotenv==1.1.1 requests==2.32.5 requests-cache==1.2.1 +saxonche==12.5.0 typing_extensions==4.15.0 url-normalize==2.2.1 urllib3==2.5.0 From 732e6ef586c4c5083b603409cda9d3f48055c8c1 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 16:45:21 -0500 Subject: [PATCH 017/222] rewrite exec_xsl method to call python script, initialize cmd as array --- lib/datura/file_type.rb | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 2e53fd174..ad372131b 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -189,24 +189,33 @@ def add_xsl_params_options # TODO can remove most of these parameters and grab them from instance variables def exec_xsl(input, xsl, ext, outpath=nil, params=nil) - saxon_params = CommonXml.stringify_params(params) - cmd = "saxon -s:#{input} -xsl:#{xsl}" - # TODO which way would we rather do this? - # cmd << " -o:#{outpath}/#{filename(false)}.#{ext}" if outpath - cmd << " #{saxon_params}" - cmd << " | tee #{outpath}/#{filename(false)}.#{ext}" if outpath - puts "using command #{cmd}" if @options["verbose"] - Open3.popen3(cmd) do |stdin, stdout, stderr| - out = stdout.read - err = stderr.read - if err.length > 0 - msg = "There was an error transforming #{filename}: #{err}" - return { "error" => msg } - else - puts "Successfully transformed #{filename}" - return { "doc" => out } + # build the python script path + python_script = File.join( + @options["datura_dir"], "lib", "datura", "python", "xslt_transform.py" + ) + # initialize the command as an array, then apppend xslt params + cmd = ["python3", python_script, "--input", input, "--xsl", xsl] + if params + params.each do |k, v| + cmd += ["--param", "#{k}=#{v}"] end end + # append output path + if outpath + cmd += ["--output", "#{outpath}/#{filename(false)}.#{ext}"] + end + puts "using command #{cmd.inspect}" if @options["verbose"] + # run the command + out, err, status = Open3.capture3(*cmd) + + # check for errors + if !status.success? || err.length > 0 + msg = "There was an error transforming #{filename}: #{err}" + return { "error" => msg } + else + puts "Successfully transformed #{filename}" + return { "doc" => out } + end end def pretty_json(json) From 5e356afead70d98265541ffaabdca0fed0f15aa0 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 30 Apr 2026 16:52:39 -0500 Subject: [PATCH 018/222] create xslt transformation file with saxonche --- lib/datura/python/xslt_transform.py | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 lib/datura/python/xslt_transform.py diff --git a/lib/datura/python/xslt_transform.py b/lib/datura/python/xslt_transform.py new file mode 100644 index 000000000..1c429e50d --- /dev/null +++ b/lib/datura/python/xslt_transform.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import argparse +import sys +from pathlib import Path + + +def parse_args(): + # create the parser + parser = argparse.ArgumentParser(description="XSLT transform using saxonche") + # declare the arguments + parser.add_argument("--input", required=True, help="Path to source XML file") + parser.add_argument("--xsl", required=True, help="Path to XSL stylesheet") + parser.add_argument("--output", required=False, help="Path to write output file") + parser.add_argument("--param", action="append", default=[], metavar="KEY=VALUE", + help="XSL parameter (repeatable)") + # parse sys.argv, validate args, return namespace object with attributes + return parser.parse_args() + + +def run_transform(input_path, xsl_path, params, output_path=None): + # import here rather than at top so error is raised at call time with clear traceback if saxonche isn't installed + import saxonche + # create Saxon processor using Home Edition tier + with saxonche.PySaxonProcessor(license=False) as proc: + # create XSLT 3.0 processor + xslt_proc = proc.new_xslt30_processor() + # iterate list of kv strings + for kv in params: + if "=" not in kv: + raise ValueError(f"Invalid param format (expected KEY=VALUE): {kv!r}") + key, value = kv.split("=", 1) + xslt_proc.set_parameter(key, proc.make_string_value(value)) + # parse and compile xsl file into executable + executable = xslt_proc.compile_stylesheet(stylesheet_file=xsl_path) + # run transformation and return output as string + result = executable.transform_to_string(source_file=input_path) + # return error or write output to disk + if result is None: + raise RuntimeError("Transformation produced no output") + if output_path: + Path(output_path).write_text(result, encoding="utf-8") + print(result, end="") + + +def main(): + args = parse_args() + # catch exceptions + try: + run_transform( + input_path=args.input, + xsl_path=args.xsl, + params=args.param, + output_path=args.output, + ) + except Exception as e: + print(f"XSLT transformation error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file From 3e910480ba07948cc7f62ae66b2cc7874ea2e53a Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 09:50:03 -0500 Subject: [PATCH 019/222] shift cmd string to array, refactor tee to Ruby write --- lib/datura/common_xml.rb | 9 +++------ lib/datura/file_type.rb | 16 +++++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/datura/common_xml.rb b/lib/datura/common_xml.rb index 8fb7465db..514ae75ed 100644 --- a/lib/datura/common_xml.rb +++ b/lib/datura/common_xml.rb @@ -47,12 +47,9 @@ def self.create_xml_object(filepath, remove_ns=true) # saxon accepts params in following manner # fw=true pb=true figures=false - def self.stringify_params(param_hash) - params = "" - if param_hash - params = param_hash.map{ |k, v| "#{k}=#{v}" }.join(" ") - end - params + def self.arrayify_params(param_hash) + return [] unless param_hash + param_hash.map { |k, v| "#{k}=#{v}"} end def self.sub_corrections(aXml) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 2e53fd174..05036ed03 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -189,20 +189,22 @@ def add_xsl_params_options # TODO can remove most of these parameters and grab them from instance variables def exec_xsl(input, xsl, ext, outpath=nil, params=nil) - saxon_params = CommonXml.stringify_params(params) - cmd = "saxon -s:#{input} -xsl:#{xsl}" - # TODO which way would we rather do this? - # cmd << " -o:#{outpath}/#{filename(false)}.#{ext}" if outpath - cmd << " #{saxon_params}" - cmd << " | tee #{outpath}/#{filename(false)}.#{ext}" if outpath + # build array out of params hash + saxon_params = CommonXml.arrayify_params(params) + args = ["saxon","-s:#{input}", "-xsl:#{xsl}"] + saxon_params puts "using command #{cmd}" if @options["verbose"] - Open3.popen3(cmd) do |stdin, stdout, stderr| + Open3.popen3(*args) do |stdin, stdout, stderr| out = stdout.read err = stderr.read if err.length > 0 msg = "There was an error transforming #{filename}: #{err}" return { "error" => msg } else + # change previous tee handling to Ruby write + # TODO: we may want to consider binwrite instead to avoid any possible encoding issues + if outpath + File.write("#{outpath}/#{filename(false)}.#{ext}", out) + end puts "Successfully transformed #{filename}" return { "doc" => out } end From b18babd7ff7867330a6d0cbfa9dccee71347dd3c Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 10:59:42 -0500 Subject: [PATCH 020/222] shift YAML.load_file to YAML.safe_load --- lib/datura/elasticsearch/index.rb | 2 +- lib/datura/options.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 09828d82f..fe5e1b890 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -25,7 +25,7 @@ def initialize(options = nil, schema_mapping: false) @mapping_url = File.join(@index_url, "_mapping?pretty=true") # yaml settings (if exist) and mappings - @requested_schema = YAML.load_file(@options["es_schema"]) + @requested_schema = YAML.safe_load(File.read(@options["es_schema"]), permitted_classes: [Symbol]) @auth_header = Datura::Helpers.construct_auth_header(@options) # if requested, grab the mapping currently associated with this index # otherwise wait until after the requested schema is loaded diff --git a/lib/datura/options.rb b/lib/datura/options.rb index c478ced42..4987069b2 100644 --- a/lib/datura/options.rb +++ b/lib/datura/options.rb @@ -64,7 +64,7 @@ def read_all_configs(general, collection) def read_config(path) if File.file?(path) begin - return YAML.load_file(path) + return YAML.safe_load(File.read(path), permitted_classes: [Symbol]) rescue Exception => e puts "There was an error reading config file #{path}: #{e.message}" end From 276ea2409279213c649cd56633e730ab64278346 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 11:21:05 -0500 Subject: [PATCH 021/222] shift from interpolation to nokogiri builder api in the unlikely event of badly intended csv data --- lib/datura/file_types/file_csv.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index a2bebbb5f..9d2954ab4 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -55,7 +55,11 @@ def row_to_es(headers, row) # operates with no logic on the fields def row_to_solr(doc, headers, row) headers.each do |column| - doc.add_child("#{row[column]}") if row[column] + next unless row[column] + field = Nokogiri::XML::Node.new("field", doc) + field["name"] = column + field.content = row[column] + doc.add_child(field) end doc end From 86534fd0a3946af54db166b9f17bd7e784359505 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 1 May 2026 13:36:37 -0500 Subject: [PATCH 022/222] clean up residual todos --- lib/datura/data_manager.rb | 1 - lib/datura/parser_options/post.rb | 1 - lib/datura/solr_poster.rb | 3 --- 3 files changed, 5 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 24f4898e9..5bed3b5a5 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -109,7 +109,6 @@ def allowed_files(all_files) files end - # TODO should this move to Options class? def assert_option(opt) if !@options.key?(opt) puts "Option #{opt} was not found! Check config files and add #{opt} to continue".red diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index b6e48880f..aeab69811 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -61,7 +61,6 @@ def self.post_params puts "'today', date (2015-01-01), or date and time (2015-01-01T18:24)".light_yellow exit else - # TODO should verify that this is a correct date and turn it into a time object datetime = timify(input) if datetime.nil? exit diff --git a/lib/datura/solr_poster.rb b/lib/datura/solr_poster.rb index eb4434a88..60d3681ec 100644 --- a/lib/datura/solr_poster.rb +++ b/lib/datura/solr_poster.rb @@ -13,8 +13,6 @@ def initialize(url, commit=true) end end - # TODO this is very similar to the below _by_regex function - # so could stick them together later def clear_index del_str = "*:*" res = post_xml(del_str) @@ -65,7 +63,6 @@ def post(content, type) # post_file # assumes xml file because of historical usage of this script - # TODO refactor? def post_file(file_location) file = IO.read(file_location) post_xml(file) From c596e8f8ead831bb0eb94dddbe23fa5e9981443a Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 5 May 2026 11:25:37 -0500 Subject: [PATCH 023/222] comment out breakpoints, add sys.exit instead as appropriate --- lib/datura/python/api_fields.py | 9 ++++++--- lib/datura/python/json_to_omeka.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 9273fee9b..b5b3669cd 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -71,8 +71,11 @@ def build_item_dict(json, existing_item): update_item_value(built_item, "dh:annotationsText", fields.annotationsText(json)) update_item_value(built_item, "dh:itemText", fields.itemText(json)) return built_item - except ValueError: - breakpoint() + except ValueError as e: + #breakpoint() + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + #TODO change item linking for JSON and new API def link_item(json_item, existing_item): @@ -242,7 +245,7 @@ def get_omeka_ids(lookup_values, filter_property, item_set_id = None): if match["total_results"] >= 1: if match["total_results"] > 1: print(f"warning: multiple matches for {lookup_value}, taking first match") - breakpoint() + #breakpoint() omeka_id = match['results'][0]["o:id"] omeka_ids.append(omeka_id) else: diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 5d2f63d8d..03528a7fe 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -21,11 +21,14 @@ def post_items(pathlist): for json_item in json_items: try: if not json_item["identifier"]: - breakpoint() + #breakpoint() print("skipping item without identifier") continue except TypeError as e: - breakpoint() + #breakpoint() + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + matching_items = omeka.omeka_auth.filter_items_by_property(filter_property = "dcterms:identifier", filter_value = json_item["identifier"], item_set_id=item_set_id) if matching_items: #if item exists, update item @@ -67,7 +70,7 @@ def link_item(json_item, matching_items): print(str(err)) traceback.print_exc print(f"Error updating item {item_id}") - breakpoint() + #breakpoint() pass def add_new_item(json_item, template_number): @@ -77,14 +80,16 @@ def add_new_item(json_item, template_number): print(f"creating item {new_item['dcterms:identifier'][0]['@value']}") except KeyError as err: print(err) - breakpoint() + #breakpoint() + sys.exit(1) payload = omeka.prepare_item_payload_using_template(new_item, template_number) # add item set try: omeka.omeka_auth.add_item(payload, template_id=template_number, item_set_id=item_set_id, is_public=is_public) except Exception as err: print(err) - breakpoint() + #breakpoint() + sys.exit(1) else: print(f"error preparing item {json_item['identifier']}") @@ -97,7 +102,8 @@ def update_existing_item(json_item, matching_items): omeka.omeka_auth.update_resource(updated_item, "items") except Exception as err: print(err) - breakpoint() + #breakpoint() + sys.exit(1) #look for the output folder: /output/development/es and get all items json_dir = omeka.get_dir("output/development/es") From 079fd31f6c18a913274f40d8c7f9d792ed1d8238 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 5 May 2026 11:27:22 -0500 Subject: [PATCH 024/222] shift byebug to development dependency --- datura.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datura.gemspec b/datura.gemspec index 3f10d4769..86ba8785b 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -61,7 +61,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "nokogiri", "~> 1.10" spec.add_runtime_dependency "rest-client", "~> 2.1" spec.add_runtime_dependency "pdf-reader", "~> 2.12" - spec.add_runtime_dependency "byebug", "~> 11.0" + spec.add_development_dependency "byebug", "~> 11.0" spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" From 2d0871fff4959ebd44fbc9a453e5e86659ffb836 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 5 May 2026 11:30:11 -0500 Subject: [PATCH 025/222] shift omeka-s-tools pkg to cdrh fork --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9cb662327..d70b2a50f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ charset-normalizer==3.4.3 dotenv==0.9.9 idna==3.10 Markdown==3.8.2 -omeka-s-tools==0.3.0 +omeka_s_tools @ git+https://github.com/CDRH/omeka_s_tools.git@will_changes packaging==25.0 platformdirs==4.4.0 python-dotenv==1.1.1 From 1f9889e42119912a12c4f5f74ddc42ce04b9012a Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 5 May 2026 16:05:23 -0500 Subject: [PATCH 026/222] remove unused variables --- lib/datura/data_manager.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 9e2d72355..9ce1022d6 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -282,7 +282,7 @@ def prepare_xslt puts "Copying datura XSLT default scripts into collection" begin FileUtils.cp_r(datura_xslt, dest) - rescue Errno::ENOENT => e + rescue Errno::ENOENT raise "Could not copy XSLT scripts into the collection. " \ "Confirm you are running this command from the root of the collection repository, " \ "not from a subdirectory." @@ -308,7 +308,7 @@ def set_up_services begin # set up elasticsearch instance @es = Datura::Elasticsearch::Index.new(@options, schema_mapping: true) - rescue Errno::ECONNREFUSED, SocketError, Errno::ETIMEDOUT => e + rescue Errno::ECONNREFUSED, SocketError, Errno::ETIMEDOUT msg = "Could not connect to Elasticsearch at #{File.join(@options['es_path'], @options['es_index'])}. " \ "Confirm you have specified the correct environment " \ "(currently: #{@options['environment']}). Use -e to specify an environment." From 5bc67a9cf9ce3535f3853f5c9f18f1672d549d40 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 6 May 2026 09:00:46 -0500 Subject: [PATCH 027/222] remove saxon references from docs, add python instructions --- README.md | 4 +-- docs/4_developers/saxon.md | 67 -------------------------------------- docs/troubleshooting.md | 2 +- 3 files changed, 3 insertions(+), 70 deletions(-) delete mode 100644 docs/4_developers/saxon.md diff --git a/README.md b/README.md index 550142d6f..5177e65f2 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ source "https://rubygems.org" gem "datura", git: "https://github.com/CDRH/datura.git", tag: "v0.0.0" ``` -If this is the first datura repository on your machine, install saxon as a system wide executable. [Saxon setup documentation](docs/4_developers/saxon.md). +If this is the first datura repository on your machine, you will need to install Python 3 and `saxonche`. `saxonche` is included in `requirements.txt`, so you can install it by following the Omeka setup [instructions](/docs/1_setup/omeka_setup.md#enabling-a-virtual-environment) for installing Python dependencies. -Then, in the directory with the Gemfile, run the following: +After that, in the directory with the Gemfile, run the following: ``` gem install bundler diff --git a/docs/4_developers/saxon.md b/docs/4_developers/saxon.md deleted file mode 100644 index 303295bea..000000000 --- a/docs/4_developers/saxon.md +++ /dev/null @@ -1,67 +0,0 @@ -**Contents** -- [Install System JAR](#install-system-jar) -- [Bash Executable](#bash-executable) -- [Reference](#reference) - -A system Saxon JAR is used with scripting via a [Bash executable](#bash-executable) - -## Install System JAR - -Please use the same version of Saxon used by Oxygen as noted in -server documentation tracking versions of Saxon used across servers: -https://github.com/CDRH/cdrh-technical-documentation/blob/main/pages/Saxon.md#versions - -```bash -# Make directory named for the version: -sudo mkdir /usr/local/share/saxon-(version) - -# Symlink to versioned directory for quick switching in the future -sudo ln -s /usr/local/share/saxon-(version) /usr/local/share/saxon -``` - -Copy the Saxon JAR file (e.g. `saxon9he.jar`) from another server with `scp` or `rsync` - -or - -Download Saxon-HE from [Saxonica Open Source](https://sourceforge.net/projects/saxon/files/Saxon-HE/) - -Find saxon9he.jar and move it to usr/local/share/saxon-(version) (make sure to have the Finder show hidden files, if you are on a Mac). - -## Bash Executable -This bash executable passes its arguments through to and runs the Saxon JAR - -`sudo vim /usr/local/bin/saxon`: -```bash -#!/usr/bin/env bash - -JAVA_BIN="$(which java)" -SAXON_JAR="/usr/local/share/saxon/saxon9he.jar" - -# If JAVA_HOME is defined, call java from ENV-based path -if [[ -n "${JAVA_HOME}" ]]; then - JAVA_BIN="${JAVA_HOME}/bin/java" -fi - -# Pass all bash arguments ($@) through to the Saxon JAR -exec "${JAVA_BIN}" -jar "${SAXON_JAR}" "$@" - -# Exit with an error status if the exec shell exits back to this script -exit 1 -``` - -Add the executable permission to the Bash script:
-`sudo chmod +x /usr/local/bin/saxon` - -If you locate your Saxon JAR somewhere other than `/usr/local/share/saxon/saxon9he.jar`, make sure to update the path stored in the `SAXON_JAR` variable at the top of the script. - -Now you and others may run `saxon (arguments)` from anywhere on your system to save typing and run Saxon within scripts. - -If you get an error about locating a Java Runtime, this is probably because macOS no longer comes installed with a Java Runtime Environment. Run `brew install java`. - - -## Reference -- Download / Support Info: http://saxonica.com/download/opensource.xml -- Release Announcements: https://saxonica.plan.io/news/1 -- Bug Fixes: In release(version #).txt file on [SourceForge](https://sourceforge.net/projects/saxon/files/Saxon-HE/) -- Change Info: http://www.saxonica.com/documentation/#!changes/serialization/9.2-9.3 -- Issues: https://saxonica.plan.io/projects/saxon/issues diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 78067b0a0..aebe1bd38 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -22,7 +22,7 @@ Do you have the correct permissions to run all of the files? Check that the TEI ### XSLT -If you type `saxon` into the command line and hit enter, does it find the command? Saxon will need to be able to run from the command line. See the [saxon docs](saxon.md) for setup information. +Check that Python 3 is available and `saxonche` has been installed (`pip3 install -r requirements.txt`). ### Tests From 45e2b392960343d5eb59a355a2e716fd892d5872 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 6 May 2026 09:02:16 -0500 Subject: [PATCH 028/222] remove deprecated method previously used for saxon call --- lib/datura/common_xml.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/datura/common_xml.rb b/lib/datura/common_xml.rb index 8fb7465db..e55098fcd 100644 --- a/lib/datura/common_xml.rb +++ b/lib/datura/common_xml.rb @@ -45,16 +45,6 @@ def self.create_xml_object(filepath, remove_ns=true) file_xml end - # saxon accepts params in following manner - # fw=true pb=true figures=false - def self.stringify_params(param_hash) - params = "" - if param_hash - params = param_hash.map{ |k, v| "#{k}=#{v}" }.join(" ") - end - params - end - def self.sub_corrections(aXml) # sub .* for [.*] xml = aXml.dup From d49b35bfb9108a1e27dd663547544092da9f7bad Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 10:38:49 -0500 Subject: [PATCH 029/222] add text check to itemText method --- lib/datura/python/field_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 9dc1e851e..a4c42ff34 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -213,7 +213,7 @@ def annotationsText(self, json): def itemText(self, json): text = json.get("text", None) - if json.get("data_type"): + if text and json.get("data_type"): text += (" " + self.identifier(json)) return text From 0e766617ed2020ba061e561a1084958082b7f769 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 10:44:59 -0500 Subject: [PATCH 030/222] simplify overrides example file --- lib/datura/python/omeka_overrides_example.py | 209 ++----------------- 1 file changed, 23 insertions(+), 186 deletions(-) diff --git a/lib/datura/python/omeka_overrides_example.py b/lib/datura/python/omeka_overrides_example.py index 58d16e828..1c1a93262 100644 --- a/lib/datura/python/omeka_overrides_example.py +++ b/lib/datura/python/omeka_overrides_example.py @@ -3,191 +3,28 @@ from field_definitions import FieldDefinitions class CustomFields(FieldDefinitions): - #these are the default field definitions, which may be overridden in specific projects - def title(self, json): - return json.get("title", None) - - def identifier(self, json): - return json.get("identifier", None) + """ + Override only the methods whose behavior differs from the defaults in + FieldDefinitions. The following patterns address common override categories. + """ - def collection(self, json): - return json.get("collection", None) - - def category(self, json): - return json.get("category", None) - - def category2(self, json): - return json.get("category2", None) - - def uriData(self, json): - return json.get("uri_data", None) - - def dcterms_type(self, json): - #note that "type" is a builtin function in Python - return json.get("type", None) - - def creator(self, json): - creator_names = [creator['name'] for creator in json.get("creator") or [] if 'name' in creator] - return creator_names - - def contributor(self, json): - contributor_names = [contributor['name'] for contributor in json.get("contributor") or [] if 'name' in contributor] - return contributor_names - - def date(self, json): - return json.get("date", None) - - def description(self, json): - return json.get("description", None) - - def dcterms_format(self, json): - #note that "format" is a builtin function in Python - return json.get("format", None) - - def relation(self, json): - relation_ids = [relation['id'] for relation in json.get("has_relation") or [] if 'name' in relation] - return relation_ids - - #citation fields - #TODO is citation always single-valued? if array might need to add code to deal with that - - def publisher(self, json): - return json.get("citation", {}).get("publisher", None) - - def biblID(self, json): - #note: this field is not yet implemented in the schema - return json.get("citation", {}).get("id", None) - - def biblTitle(self, json): - return json.get("citation", {}).get("title", None) - - def biblPubPlace(self, json): - return json.get("citation", {}).get("pubplace", None) - - def issue(self, json): - return json.get("citation", {}).get("issue", None) - - def pageStart(self, json): - return json.get("citation", {}).get("page_start", None) - - def pageEnd(self, json): - return json.get("citation", {}).get("page_end", None) - - def section(self, json): - return json.get("citation", {}).get("section", None) - - def volume(self, json): - return json.get("citation", {}).get("volume", None) - - def biblTitleA(self, json): - return json.get("citation", {}).get("title_a", None) - - def biblTitleM(self, json): - return json.get("citation", {}).get("title_m", None) - - def biblTitleJ(self, json): - return json.get("citation", {}).get("title_j", None) - - def rightsHolder(self, json): - return json.get("rights_holder", None) - - def license(self, json): - return json.get("rights", None) - - def subject(self, json): - return json.get("subjects", None) - - def topic(self, json): - return json.get("topics", None) - - def category3(self, json): - return json.get("category3", None) - - def category4(self, json): - return json.get("category4", None) - - def category5(self, json): - return json.get("category5", None) - - def note(self, json): - return json.get("notes", None) - - def abstract(self, json): - return json.get("abstract", None) - - def keyword(self, json): - return json.get("keywords", None) - - def keyword2(self, json): - return json.get("keywords2", None) - - def keyword3(self, json): - return json.get("keywords3", None) - - def keyword4(self, json): - return json.get("keywords4", None) - - def keyword5(self, json): - return json.get("keywords5", None) - - def source(self, json): - return json.get("has_source") and json.get("has_source", {}).get("title") + # Pattern 1: read from a different ES key + # def title(self, json): + # return json.get("preferred_title") or json.get("title") - - def medium(self, json): - return json.get("medium", None) - - def extent(self, json): - return json.get("extent", None) - - def language(self, json): - return json.get("language", None) - - def box(self, json): - return json.get("container_box", None) - - def folder(self, json): - return json.get("container_folder", None) - - def name(self, json): - person_names = [person['name'] for person in json.get("person") or [] if 'name' in person] - return person_names - - def spatial_short_name(self, json): - places = [json["spatial"]] if isinstance(json["spatial"], dict) else json["spatial"] - if places: - place_names = [place['short_name'] for place in places or [] if 'short_name' in place] - return place_names - - def correspSentName(self, json): - return json.get("correspSentName_omeka_s", None) - - def correspSentPlace(self, json): - return json.get("correspSentPlace_omeka_s", None) - - def correspSentDate(self, json): - return json.get("correspSentDate_omeka_s", None) - - def correspDeliveredName(self, json): - return json.get("correspDeliveredName_omeka_s", None) - - def correspDeliveredPlace(self, json): - return json.get("correspDeliveredPlace_omeka_s", None) - - def correspDeliveredDate(self, json): - return json.get("correspDeliveredDate_omeka_s", None) - - def distributor(self, json): - return json.get("distributor_omeka_s", None) - - def authority(self, json): - return json.get("authority_omeka_s", None) - - def biblNote(self, json): - return json.get("biblNote_omeka_s", None) - - def annotationsText(self, json): - return json.get("annotations_text", None) - - def itemText(self, json): - return json.get("text", None) + # Pattern 2: citation sub-field (base class handles null/array automatically) + # def publisher(self, json): + # return self._get_citation(json).get("publisher", None) + + # Pattern 3: combine multiple ES fields into one Omeka value + # def creator(self, json): + # creators = json.get("creator") or [] + # return [ + # f"{c['name']} ({c['role']})" if c.get("role") else c["name"] + # for c in creators if c.get("name") + # ] + + # Pattern 4: transform a value (e.g. reformat a date or strip whitespace) + # def dateDisplay(self, json): + # raw = json.get("date_display", None) + # return raw.strip() if raw else None \ No newline at end of file From b7a5fb15592d6c6ae48c81fb002df18260496b36 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 11:07:59 -0500 Subject: [PATCH 031/222] add method to check config for req omeka params --- lib/datura/data_manager.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 24f4898e9..280c77904 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -64,6 +64,13 @@ def load_collection_classes end end + def check_omeka_options + %w[omeka_server key_identity key_credential iiif_server + resource_template omeka_data_base item_set].each do |opt| + assert_option(opt) + end + end + def print_options pretty = JSON.pretty_generate(@options) puts "Options: #{pretty}" From 126afdb1001b7b237ba6d35e5135d898ca90daac Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 11:11:20 -0500 Subject: [PATCH 032/222] move DataManager gen out of conditional so data can be validated before run --- bin/post_omeka | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/post_omeka b/bin/post_omeka index bba61370a..d184667ec 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -37,9 +37,11 @@ end optparse.parse(ARGV) #add options to output a json file instead of posting it to Elasticsearch ARGV.unshift("-x", "es", "-o", "-t") +#create and validate DataManager before conditional run +manager = Datura::DataManager.new +manager.check_omeka_options #exit with clear error if any key is missing #skip generation step with option -s if generate_es - manager = Datura::DataManager.new manager.run end datura_dir = File.join(File.dirname(__FILE__), "..") From c7a4f4f4734ea9ecd3a11a22de1c65cbc1cf0d6b Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 11:13:28 -0500 Subject: [PATCH 033/222] move DataManager gen out of conditional in post_omeka_html --- bin/post_omeka_html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 27f3a3522..35a2bdea6 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -39,9 +39,11 @@ optparse.parse(ARGV) ARGV.delete("-m") #add option to generate html ARGV.unshift("-x", "html") +#create and validate DataManager before conditional run +manager = Datura::DataManager.new +manager.check_omeka_options #exit with clear error if any key is missing #skip generation step with option -s if generate_es - manager = Datura::DataManager.new manager.run end From 424a76768185b7c712d4c81b172debb0e688a784 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 11:20:56 -0500 Subject: [PATCH 034/222] remove byebug from dependency list; moved to dev dependency --- bin/post_omeka | 1 - bin/post_omeka_html | 1 - lib/datura/data_manager.rb | 1 - 3 files changed, 3 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index d184667ec..4fe60b92f 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -1,7 +1,6 @@ #!/usr/bin/env ruby require "datura" -require "byebug" require "optparse" require "shellwords" diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 35a2bdea6..7955d2b02 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -1,7 +1,6 @@ #!/usr/bin/env ruby require "datura" -require "byebug" require "shellwords" generate_es = true diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 280c77904..cd40f6d88 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -1,7 +1,6 @@ require "colorize" require "logger" require "yaml" -require "byebug" require_relative "./requirer.rb" class Datura::DataManager From ee31168741628d99b48e659a7af9b954cf0c7838 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 12:54:21 -0500 Subject: [PATCH 035/222] add sys import --- lib/datura/python/api_fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index b5b3669cd..c2e55e6c0 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -1,5 +1,6 @@ import json import re +import sys import omeka from datetime import datetime from field_definitions import get_fields From 87457905906f363470076f9a8c21f56dcbfa8e08 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 12:55:44 -0500 Subject: [PATCH 036/222] add parens to traceback.print_exc --- lib/datura/python/json_to_omeka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 03528a7fe..bf572643e 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -68,7 +68,7 @@ def link_item(json_item, matching_items): omeka.omeka_auth.update_resource(linked_item, "items") except Exception as err: print(str(err)) - traceback.print_exc + traceback.print_exc() print(f"Error updating item {item_id}") #breakpoint() pass From 75824bd17aa2fdcdf00abc2f8ce5799d694cde21 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 12:56:55 -0500 Subject: [PATCH 037/222] fix comparison typo --- lib/datura/python/omeka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 5b079d04f..dc7c53227 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -46,7 +46,7 @@ def get_item_set(): elif env == "development": item_set = dev_config["item_set"] else: - item_set == None + item_set = None return item_set def get_environment(): From d025081fdef40e1c4bec1749ec44eddbd569d410 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 13:05:09 -0500 Subject: [PATCH 038/222] add comment to note non-working function for later refactoring --- lib/datura/python/omeka.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index dc7c53227..97a8dcfd1 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -25,6 +25,7 @@ def get_config(path, env='default'): print(exc) def reset(): + # TODO: this function is currently broken; local variables do not overwrite module-level globals omeka = OmekaAPIClient(config['omeka_server']) omeka_auth = OmekaAPIClient( api_url = config['omeka_server'], From 5ad0fae3b23130964853d14d0988fd1f30dd4761 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 13:09:03 -0500 Subject: [PATCH 039/222] add conditional to skip any empty HTML content --- lib/datura/python/html_and_media_ingest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 6dbbce353..870646c78 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -80,6 +80,9 @@ def ingest_html(json_item, matching_item): try: with open(file_path, "r") as file: html_content = file.read() + if not html_content.strip(): + print(f"HTML file for {json_item['identifier']} is empty, skipping") + return media_payload = { "o:is_public": True, "data": { From bd46097e37973813d49619058db023154b495ab4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 15:42:11 -0500 Subject: [PATCH 040/222] add SSL verification to Net::HTTP (Solr) and get_url helper --- lib/datura/helpers.rb | 9 +++++++-- lib/datura/solr_poster.rb | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index bcc245fff..87f7a3796 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -74,8 +74,13 @@ def self.get_input(original_input, msg) # get_url # sends a request to a given url def self.get_url(url) - url = URI.parse(url) - Net::HTTP.get_response(url) + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + if uri.scheme == "https" + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + end + http.request(Net::HTTP::Get.new(uri.request_uri)) end # make_dirs diff --git a/lib/datura/solr_poster.rb b/lib/datura/solr_poster.rb index eb4434a88..84a21861a 100644 --- a/lib/datura/solr_poster.rb +++ b/lib/datura/solr_poster.rb @@ -55,7 +55,8 @@ def commit_solr def post(content, type) url = URI.parse(@url) http = Net::HTTP.new(url.host, url.port) - http.use_ssl = @url[/^https/] ? true : false + http.use_ssl = @url.start_with?("https") + http.verify_mode = OpenSSL::SSL::VERIFY_PEER http.open_timeout = 10 request = Net::HTTP::Post.new(url.request_uri) request.body = content From 1df43c7abe7ff73f3a5a6bcccd29d6ee78f52c75 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 7 May 2026 15:49:00 -0500 Subject: [PATCH 041/222] add warning when user and password are set and es_path is http --- lib/datura/helpers.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 87f7a3796..8484e6b0f 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -142,6 +142,12 @@ def self.should_update?(file, since_date=nil) def self.construct_auth_header(options) username = options["es_user"] password = options["es_password"] + + if (username || password) && options["es_path"]&.start_with?("http://") + warn "[SECURITY WARNING] ES credentials are set but es_path uses unencrypted HTTP. " \ + "Credentials will be transmitted in cleartext. Use HTTPS in production." + end + { "Authorization" => "Basic #{Base64::encode64("#{username}:#{password}")}" } end From b283e2df62ebf291193ea9d535222522a653e6b2 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 13:05:31 -0500 Subject: [PATCH 042/222] remove unneeded packages --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d70b2a50f..7172dfdb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,7 @@ attrs==25.3.0 cattrs==25.1.1 certifi==2025.8.3 charset-normalizer==3.4.3 -dotenv==0.9.9 idna==3.10 -Markdown==3.8.2 omeka_s_tools @ git+https://github.com/CDRH/omeka_s_tools.git@will_changes packaging==25.0 platformdirs==4.4.0 From b96ae406d72f3bccad257567ea39ea3d758f64ee Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 15:55:46 -0500 Subject: [PATCH 043/222] new file with OmekaContext class, custom exception hierarchy, and logging setup --- lib/datura/python/omeka_context.py | 451 +++++++++++++++++++++++++++++ 1 file changed, 451 insertions(+) create mode 100644 lib/datura/python/omeka_context.py diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py new file mode 100644 index 000000000..661001b81 --- /dev/null +++ b/lib/datura/python/omeka_context.py @@ -0,0 +1,451 @@ +""" +omeka_context.py + +Central context object and exception hierarchy for the Omeka S ingestion pipeline. + +""" + +import logging +import sys +from pathlib import Path +from typing import Dict, List, Optional + +import yaml +from omeka_s_tools.api import OmekaAPIClient + +# Module-level logger. Using __name__ means log records from this module +# appear as "omeka_context" in the output, making it easy to filter. +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Logging setup +# --------------------------------------------------------------------------- + +def configure_logging(level="INFO"): + """ + Configure the root logger for the pipeline. + + Should be called once at the very start of each entrypoint script before + any other work begins. Subsequent calls are safe but have no additional + effect — Python's logging.basicConfig() is a no-op if handlers are already + attached to the root logger. + + Parameters: + * level - logging level string: "DEBUG", "INFO", "WARNING", or "ERROR". + Defaults to "INFO". Use "DEBUG" to trace individual API calls + and property ID cache hits/misses. + """ + logging.basicConfig( + format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", + level=getattr(logging, level.upper(), logging.INFO), + ) + + +# --------------------------------------------------------------------------- +# Exception hierarchy +# --------------------------------------------------------------------------- + +class OmekaError(Exception): + """ + Base class for all Omeka pipeline errors. + + Catch this to handle any pipeline error generically, or catch a subclass + to handle a specific failure mode. All subclasses produce a descriptive + human-readable message so that log entries are self-explanatory without + requiring a full traceback. + """ + + +class OmekaConfigError(OmekaError): + """ + Raised when the pipeline cannot start due to a configuration problem. + + Common causes: + - config/private.yml does not exist in the collection directory + - The YAML file is malformed and cannot be parsed + - A required key (e.g. "omeka_server", "key_identity") is absent + + This is always a fatal error: the pipeline cannot connect to or + authenticate with Omeka S without a valid configuration, so the process + exits immediately rather than attempting to continue. + """ + + +class OmekaAPIError(OmekaError): + """ + Raised when an Omeka S API call fails for a specific item or resource. + + Unlike OmekaConfigError, this is typically a per-item failure. The run + continues and the error is accumulated via ctx.record_error() so that + a full summary is printed at the end of the run. + + Attributes: + * identifier - the CDRH identifier string of the item that failed, + or "unknown" if the identifier could not be determined + * operation - short description of the failing operation, e.g. "add_item" + * cause - the original exception raised by the API client + """ + def __init__(self, identifier, operation, cause): + self.identifier = identifier + self.operation = operation + self.cause = cause + super().__init__( + "{} failed for {!r}: {}".format(operation, identifier, cause) + ) + + +class OmekaItemNotFoundError(OmekaError): + """ + Raised when an item lookup returns zero results but exactly one was expected. + + Typical causes: + - An item was not ingested during the posting pass before the linking pass ran + - An identifier was changed between runs, leaving the old Omeka record orphaned + """ + + +class OmekaMultipleMatchesError(OmekaError): + """ + Raised when an item lookup returns more than one result for a given identifier. + + Identifiers should be unique within an item set. Multiple matches indicate a + data integrity problem that must be resolved in the Omeka admin UI before the + affected item can be updated automatically. + """ + + +class OmekaMediaError(OmekaError): + """ + Raised when a media upload or deletion operation fails. + + Separated from OmekaAPIError so that callers can distinguish between failures + on item metadata (OmekaAPIError) and failures on associated media objects + (OmekaMediaError), which may warrant different recovery strategies. + """ + + +# --------------------------------------------------------------------------- +# Context +# --------------------------------------------------------------------------- + +class OmekaContext: + """ + Encapsulates all configuration and shared state for one pipeline run. + + Create exactly one OmekaContext per entrypoint invocation using the class + method from_args(). Pass the resulting context object to every pipeline + function that needs configuration or API access. + + The context holds: + - Parsed, validated configuration from config/private.yml + - A single authenticated OmekaAPIClient (self.client) + - A property ID cache to avoid repeated API lookups per field per item + - An error accumulator that collects per-item failures without halting the run + + """ + + @classmethod + def from_args(cls, args): + """ + Build an OmekaContext from a parsed argparse.Namespace. + + Loads config/private.yml from the current working directory (the + collection root), validates that all required keys are present, and + initialises the authenticated API client. + + Parameters: + * args - argparse.Namespace produced by an entrypoint's _parse_args(). + Expected attributes: + .environment str "development" or "production" + .regex str optional file-filter pattern, or None + .media_skip bool skip re-ingesting existing media + (html_and_media_ingest only; absent on + json_to_omeka args, defaults to False) + + Raises OmekaConfigError if the config file is missing, unparseable, + or is missing a required key. + """ + conf_path = Path.cwd() / "config" / "private.yml" + logger.debug("Loading config from %s", conf_path) + + # Load the top-level "default" section, which holds credentials and + # settings common to all environments. + default_config = cls._load_config(conf_path, env="default") + + # Load the environment-specific section (primarily contains item_set). + # If the section is absent (e.g. an unrecognised environment name was + # passed), log a warning and fall back to an empty dict — item_set_id + # will be None and the run will proceed without filtering by item set. + try: + env_config = cls._load_config(conf_path, env=args.environment) + except OmekaConfigError: + logger.warning( + "No config section found for environment %r; " + "item_set will be None and items will not be scoped to a set.", + args.environment, + ) + env_config = {} + + return cls( + config=default_config, + env_config=env_config, + environment=args.environment, + # getattr with a default handles entrypoints that don't define + # every flag (e.g. json_to_omeka.py has no --media-skip). + regex=getattr(args, "regex", None), + media_skip=getattr(args, "media_skip", False), + ) + + @staticmethod + def _load_config(path, env): + """ + Load a single environment section from a YAML config file. + + Parameters: + * path - pathlib.Path pointing to the YAML file + * env - the top-level key to extract, e.g. "default" or "development" + + Returns the dict for that section. + + Raises OmekaConfigError with a descriptive message on any I/O or parse + failure, so that operators know exactly what to fix without reading a + Python traceback. + """ + try: + with open(path) as f: + contents = yaml.safe_load(f) + except FileNotFoundError: + raise OmekaConfigError( + "Config file not found: {}. " + "Ensure config/private.yml exists in the collection directory " + "and that you are running the script from the collection root." + .format(path) + ) + except yaml.YAMLError as exc: + raise OmekaConfigError( + "Could not parse YAML in {}: {}".format(path, exc) + ) + + if env not in contents: + raise OmekaConfigError( + "Environment section {!r} not found in {}. " + "Available sections: {}" + .format(env, path, list(contents.keys())) + ) + + return contents[env] + + def __init__(self, config, env_config, environment, regex, media_skip): + """ + Initialise the context. Prefer OmekaContext.from_args() over calling + this constructor directly except in tests. + + Parameters: + * config - dict from the "default" section of private.yml; + must contain omeka_server, key_identity, key_credential, + resource_template, and omeka_data_base + * env_config - dict from the environment-specific section; used to + look up item_set + * environment - "development" or "production" + * regex - optional regex string to filter input file paths; + None means process all files in the output directory + * media_skip - if True, items that already have 2+ media objects + (thumbnail + HTML) are skipped during media ingest + """ + # ---- Validate required config keys -------------------------------- + # Validate up front so that failures are immediate and descriptive. + # A missing key surfaced here gives a clear error message; the same + # key missing inside a loop gives an opaque KeyError mid-run. + required_keys = [ + "omeka_server", + "key_identity", + "key_credential", + "resource_template", + "omeka_data_base", + ] + for key in required_keys: + if key not in config: + raise OmekaConfigError( + "Missing required config key {!r}. " + "Check the 'default' section of config/private.yml." + .format(key) + ) + + # ---- Runtime flags ------------------------------------------------ + self.environment = environment + self.regex = regex + self.media_skip = media_skip + + # ---- Config values ------------------------------------------------ + self.template_number = config["resource_template"] + self.omeka_data_base = config["omeka_data_base"] + # iiif_server is optional — not all collections ingest thumbnails. + self.iiif_server = config.get("iiif_server", "") + + # Keep the environment-specific dict for the item_set_id property. + self._env_config = env_config + + # ---- Credentials (stored for reset_client) ------------------------ + # Stored privately so that credential strings are not accidentally + # printed, logged, or serialised through the public interface. + self._api_url = config["omeka_server"] + self._key_identity = config["key_identity"] + self._key_credential = config["key_credential"] + + # ---- API client --------------------------------------------------- + # Single authenticated client used for all API operations. + self.client = OmekaAPIClient( + api_url=self._api_url, + key_identity=self._key_identity, + key_credential=self._key_credential, + ) + logger.debug( + "OmekaContext initialised (environment=%r, template=%s)", + self.environment, + self.template_number, + ) + + # ---- Property ID cache -------------------------------------------- + # Maps Omeka term strings (e.g. "dcterms:title") to their numeric IDs. + # IDs are stable within a single Omeka S instance for the lifetime of + # a run, so fetching each term once is sufficient. + self._property_id_cache = {} # type: Dict[str, int] + + # ---- Error accumulator -------------------------------------------- + # Non-fatal per-item errors are appended here rather than aborting the + # run. report_errors() logs a consolidated summary at the end. + self._errors = [] # type: List[OmekaError] + + # ----------------------------------------------------------------------- + # Properties + # ----------------------------------------------------------------------- + + @property + def item_set_id(self): + # type: () -> Optional[int] + """ + The Omeka item set ID for the current environment, or None. + + Stored in the environment-specific config section so that development + and production ingests target different item sets. Returns None if no + item_set key is present (e.g. running locally without a complete + private.yml, or using an environment that has no item_set configured). + """ + return self._env_config.get("item_set") + + @property + def is_public(self): + # type: () -> bool + """ + True only when environment is "production". + + Items created with is_public=False are visible only to logged-in Omeka + admins, which prevents in-progress development ingests from appearing + to public users of the site. + """ + return self.environment == "production" + + # ----------------------------------------------------------------------- + # API helpers + # ----------------------------------------------------------------------- + + def get_property_id(self, term): + # type: (str) -> int + """ + Return the numeric Omeka property ID for a term, using a per-run cache. + + The first call for a given term makes one API request and stores the + result. All subsequent calls return the cached integer immediately. + The cache is preserved across reset_client() calls because term-to-ID + mappings are stable within a single Omeka S instance. + + Parameters: + * term - Omeka property term string, e.g. "dcterms:title", "dh:collection" + """ + if term not in self._property_id_cache: + logger.debug("Fetching property ID for term %r (not yet cached)", term) + self._property_id_cache[term] = self.client.get_property_id(term) + return self._property_id_cache[term] + + def reset_client(self): + """ + Re-instantiate the authenticated API client with a fresh connection. + + Called in json_to_omeka.py between the item-posting pass and the + item-linking pass to obtain a clean session before the second round + of API requests. + + The property ID cache is intentionally preserved: term-to-ID mappings + do not change between passes, so clearing and re-fetching them would + waste API calls without any benefit. + """ + logger.debug("Resetting API client (property ID cache preserved)") + self.client = OmekaAPIClient( + api_url=self._api_url, + key_identity=self._key_identity, + key_credential=self._key_credential, + ) + + # ----------------------------------------------------------------------- + # Path resolution + # ----------------------------------------------------------------------- + + def resolve_path(self, relative): + # type: (str) -> Path + """ + Resolve a path relative to the current working directory (collection root). + Callers interpolate the environment into the path template: + + json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) + + This ensures that passing -e production reads from output/production/ + rather than always using output/development/. + + Parameters: + * relative - path string relative to cwd, e.g. "output/development/es" + + Returns an absolute pathlib.Path. + """ + return (Path.cwd() / relative).resolve() + + # ----------------------------------------------------------------------- + # Error accumulation + # ----------------------------------------------------------------------- + + def record_error(self, err): + # type: (OmekaError) -> None + """ + Record a non-fatal per-item error without halting the run. + + Use for item-level failures (API errors, missing files, malformed data) + where the correct behaviour is to log the problem, skip the affected + item, and continue processing the rest of the batch. + + Fatal errors that make the entire run impossible (wrong credentials, + missing config file) should raise OmekaConfigError directly and let + the process exit with a traceback. + + Parameters: + * err - an OmekaError (or subclass) instance describing the failure + """ + logger.error(str(err)) + self._errors.append(err) + + def report_errors(self): + """ + Log a consolidated summary of all errors recorded during the run. + + Called by entrypoint scripts just before sys.exit(). If any errors + were recorded, the entrypoint should exit with code 1 so that the + calling Ruby process (system() in bin/post_omeka or bin/post_omeka_html) + can detect that the run completed with failures. + + If no errors were recorded, logs a single success message. + """ + if self._errors: + logger.warning("Run completed with %d error(s):", len(self._errors)) + for err in self._errors: + logger.warning(" %s", err) + else: + logger.info("Run completed successfully with no errors.") From 5dc4392814bd544f036ec2258197dc4fc6e4db21 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 16:10:19 -0500 Subject: [PATCH 044/222] refactor as pure functions only, passing in ctx --- lib/datura/python/omeka.py | 354 ++++++++++++++++++++----------------- 1 file changed, 192 insertions(+), 162 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 97a8dcfd1..5c2a2a40e 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -1,179 +1,210 @@ +""" +omeka.py + +Utility functions for the Omeka S ingestion pipeline. + +""" + from pathlib import Path import json -from omeka_s_tools.api import OmekaAPIClient -import math -import yaml -import argparse import re -#needed for debugging purposes -import traceback -import os - -# custom methods - -def get_dir(relative_path): - cwd = Path.cwd() - return (cwd / relative_path).resolve() - -def get_config(path, env='default'): - with open(path) as stream: - try: - contents = yaml.safe_load(stream) - return(contents[env]) - except yaml.YAMLError as exc: - print(exc) - -def reset(): - # TODO: this function is currently broken; local variables do not overwrite module-level globals - omeka = OmekaAPIClient(config['omeka_server']) - omeka_auth = OmekaAPIClient( - api_url = config['omeka_server'], - key_identity = config['key_identity'], - key_credential = config['key_credential'] - ) -def item_sets(): - pages = math.ceil(omeka.get_resources("item_sets")["total_results"] / 5) - item_sets = [] - for i in range(pages): - item_sets += omeka.get_resources("item_sets", page=i)["results"] - return item_sets - -def get_item_set(): - env = get_environment() - if env == "production": - item_set = prod_config["item_set"] - elif env == "development": - item_set = dev_config["item_set"] - else: - item_set = None - return item_set +def add_media_to_item(ctx, item_id, media_file, payload=None, template_id=None, class_id=None): + """ + Upload a media file and associate it with an existing Omeka S item. -def get_environment(): - environment = args.environment - return environment - -def get_regex(): - regex = args.regex - return regex - -def add_media_to_item(item_id, media_file, payload={}, template_id=None, class_id=None): - # copied from the module to modify with different ingester - ''' - Upload a media file and associate it with an existing item. + This is a modified version of the omeka-s-tools library method. The key + difference is that the ingester type ("upload", "html", etc.) is read from + payload["o:ingester"] rather than always defaulting to "upload". This allows + the same function to handle both binary file uploads and the HTML ingester, + which reads content from payload["data"]["html"] instead of a file. Parameters: - * `item_id` - the Omeka id of the item this media file should be added to - * `media_path` - a path to an image/media file (string or pathlib Path) - * `payload` (optional) - metadata to attach to media object, either - a dict generated by `prepare_item_payload()` or `prepare_item_payload_using_template()`, - or a string which is used as the value for `dcterms:title`. - * `template_id` - internal Omeka identifier of a resource template you want to attach to this item - * `class_id` - internal Omeka identifier of a resource class you want to attach to this item - - Returns: - * a dict providing a JSON-LD representation of the new media object - ''' + * ctx - OmekaContext providing the authenticated API client + * item_id - numeric Omeka ID of the item this media should attach to + * media_file - path to the media file as a string or pathlib.Path. + For the HTML ingester, this is the path to the .html file, + although the Omeka API reads content from payload["data"]["html"] + rather than the uploaded bytes. + * payload - dict of metadata for the media object. Must contain + "o:ingester" (e.g. "upload" or "html") and any additional + metadata fields. Defaults to an empty dict. + * template_id - optional numeric Omeka resource template ID to attach + to the media object (rarely needed for media). + * class_id - optional numeric Omeka resource class ID. If template_id + is given and class_id is not, the class is inferred from + the template automatically. + + Returns the Omeka JSON-LD representation of the newly created media object. + """ + if payload is None: + payload = {} + files = {} - # For backwards compatibility + + # Legacy dict-style call: {"path": ..., "title": ...} + # Preserved for backwards compatibility with any callers using the older + # interface from the omeka-s-tools library. if isinstance(media_file, dict): path = media_file['path'] payload = media_file['title'] - # Make sure path is a Path object + + # Normalise the path to a pathlib.Path regardless of input type. path = Path(media_file) + + # If a bare string title was passed as the payload, wrap it in the + # standard item payload format expected by the API. if isinstance(payload, str): - payload = omeka.omeka_auth.prepare_item_payload({'dcterms:title': [payload]}) + payload = ctx.client.prepare_item_payload({'dcterms:title': [payload]}) + + # Attach resource template metadata if requested. if template_id: - payload['o:resource_template'] = omeka.omeka_auth.format_resource_id(template_id, 'resource_templates') + payload['o:resource_template'] = ctx.client.format_resource_id( + template_id, 'resource_templates' + ) if not class_id: - template = omeka.omeka_auth.get_resource_by_id(template_id, 'resource_templates') + # Infer the resource class from the template when not supplied. + template = ctx.client.get_resource_by_id(template_id, 'resource_templates') class_id = template['o:resource_class']['o:id'] if class_id: - payload['o:resource_class'] = omeka.omeka_auth.format_resource_id(class_id, 'resource_classes') - #add option to change ingester - ingester = payload["o:ingester"] if payload["o:ingester"] else "upload" + payload['o:resource_class'] = ctx.client.format_resource_id( + class_id, 'resource_classes' + ) + + # Use the ingester declared in the payload, falling back to "upload". + # Using .get() guards against a missing key + ingester = payload.get("o:ingester") or "upload" + + # Core fields required by Omeka S for any media POST. file_data = { 'o:ingester': ingester, - 'file_index': '0', - 'o:source': path.name, - 'o:item': {'o:id': item_id} + 'file_index': '0', # index into the files[] multipart array + 'o:source': path.name, # original filename, shown in Omeka admin + 'o:item': {'o:id': item_id}, } payload.update(file_data) - files[f'file[0]'] = path.read_bytes() + + # Read the raw file bytes and attach them as file[0] in the multipart body. + # For the HTML ingester, Omeka reads content from payload["data"]["html"] + # and ignores the file bytes, but including them does not cause errors. + files['file[0]'] = path.read_bytes() files['data'] = (None, json.dumps(payload), 'application/json') - response = omeka_auth.s.post(f'{omeka_auth.api_url}/media', files=files, params=omeka_auth.credentials) - data = omeka_auth.process_response(response) - return data - -def prepare_item_payload_using_template(terms, template_id): - ''' - Prepare an item payload, checking the supplied terms and values against the specified template. - Note: - * terms that are not in the template will generate a warning and be dropped from the payload - * data types that don't match the template definitions will generate a warning and the term will be dropped from the payload - * if no data type is supplied, a type that conforms with the template definition will be used - Parameters: - * `terms`: a dict of terms, values, and (optionally) data types - * `template_id`: Omeka's internal numeric identifier for the template + response = ctx.client.s.post( + '{}/media'.format(ctx.client.api_url), + files=files, + params=ctx.client.credentials, + ) + return ctx.client.process_response(response) + - Returns: - * the payload dict - ''' - template_properties = omeka_auth.get_template_properties(template_id) +def prepare_item_payload_using_template(ctx, terms, template_id): + """ + Build an item payload, validating terms and values against a resource template. + + Behaviour: + - Terms not present in the template are logged and dropped from the payload. + - Values whose data type does not match the template definition are dropped. + - If no data type is supplied for a value, the template default is used, + or "literal" if the template allows it and no single default exists. + + Parameters: + * ctx - OmekaContext providing the authenticated API client + * terms - dict mapping Omeka property term strings to lists of value + dicts, e.g. {"dcterms:title": [{"value": "My Title"}]} + * template_id - Omeka's internal numeric ID for the resource template + + Returns a payload dict suitable for passing to ctx.client.add_item(). + """ + # Fetch the template's property definitions once; this dict maps term + # strings to their allowed types and property IDs. + template_properties = ctx.client.get_template_properties(template_id) payload = {} + for term, values in terms.items(): - if term in template_properties: - property_details = template_properties[term] - payload[term] = [] - for value in values: - if not isinstance(value, dict): - value = {'value': value} - # The supplied data type doesn't match the template - if 'type' in value and value['type'] not in property_details['type']: - print(f'Data type "{value["type"]}" for term "{term}" not allowed by template') - break - elif 'type' not in value: - # Use default datatype from template if none is supplied - if len(property_details['type']) == 1: - value['type'] = property_details['type'][0] - # Use literal if allowed by template and data type not supplied - elif 'literal' in property_details['type']: - value['type'] = 'literal' - # Don't know what data type to use - else: - print(f'Specify data type for term "{term}"') - break - if "property_id" in value: - #don't format values that have already been formatted - payload[term].append(value) + if term not in template_properties: + # Terms outside the template are intentionally dropped — each + # collection defines which fields are relevant to its template. + print('Term {} not in template'.format(term)) + continue + + property_details = template_properties[term] + payload[term] = [] + + for value in values: + # Ensure value is a dict with at least a "value" key. + if not isinstance(value, dict): + value = {'value': value} + + # Validate the supplied data type against the template's allowed types. + if 'type' in value and value['type'] not in property_details['type']: + print( + 'Data type "{}" for term "{}" not allowed by template' + .format(value['type'], term) + ) + break + + if 'type' not in value: + # Infer a data type from the template definition. + if len(property_details['type']) == 1: + # Only one type allowed — use it. + value['type'] = property_details['type'][0] + elif 'literal' in property_details['type']: + # Multiple types allowed; prefer "literal" as the default. + value['type'] = 'literal' else: - # Add a value formatted according to the data type - payload[term].append(omeka_auth.prepare_property_value(value, property_details['property_id'])) - # The supplied term is not in the template - else: - print(f'Term {term} not in template') + # Cannot determine a type; skip this value. + print('Specify data type for term "{}"'.format(term)) + break + + if "property_id" in value: + # Value was already formatted by a prior call; append as-is to + # avoid double-formatting. + payload[term].append(value) + else: + # Format the value according to the template property definition. + payload[term].append( + ctx.client.prepare_property_value( + value, property_details['property_id'] + ) + ) + return payload -def prepare_property_value(value, property_id, label = ""): - ''' - Formats a property value according to its datatype as expected by Omeka. - The formatted value can be used in a payload to create a new item. - Parameters: - * `value` - a dict containing a `value` and (optionally) a `type` - * `property_id` - the numeric identifier of the property - * `label` - a text label for the URI if `type` is "uri" +def prepare_property_value(value, property_id, label=""): + """ + Format a single property value in the structure expected by Omeka S. - Note that is no `type` is supplied, 'literal' will be used by default. + This is a custom version of the omeka-s-tools library method, extended to + support an optional text label for URI-type values. It is used in + api_fields.add_formatted_value() for all standard property formatting. - Returns: - * a dict with values for `property_id`, `type`, and either `@id` or `@value`. - ''' + Parameters: + * value - a string, int, float, or dict. Non-dict values are + automatically wrapped: {"value": }. Dicts may + include a "type" key; if absent, "literal" is used. + * property_id - numeric Omeka property ID for this term + * label - display label for URI values. If omitted, the last path + segment of the URI is used as the label. + + Returns a dict formatted for inclusion in an Omeka S item payload. + + NOTE: The "resource:item" branch contains a reference to `self.api_url` + which is a pre-existing copy-paste bug from the library source (this is a + standalone function, not a method, so `self` is undefined). This branch + is not reached by any current pipeline caller — all values are "literal" + or "uri" — so the bug has been left in place with this comment rather than + silently changing potentially-load-bearing code during a refactor. + If you need resource:item linking, use ctx.client.prepare_property_value() + (the library version) instead. + """ + # Wrap bare scalars so the rest of the function can assume a dict. if not isinstance(value, dict): value = {'value': value} + # Default to "literal" when no explicit type is provided. try: data_type = value['type'] except KeyError: @@ -181,44 +212,43 @@ def prepare_property_value(value, property_id, label = ""): property_value = { 'property_id': property_id, - 'type': data_type + 'type': data_type, } if data_type == 'resource:item': - property_value['@id'] = f'{self.api_url}/items/{value["value"]}' + # BUG: `self` is not defined here. This is dead code for current callers. + # Use ctx.client.prepare_property_value() for resource:item values. + property_value['@id'] = '{}/items/{}'.format(self.api_url, value['value']) # noqa: F821 property_value['value_resource_id'] = value['value'] property_value['value_resource_name'] = 'items' elif data_type == 'uri': property_value['@id'] = value['value'] + # Fall back to the last URI segment when no explicit label is given. if label == "": property_value["o:label"] = value["value"].split("/")[-1] else: property_value["o:label"] = label else: + # "literal", "numeric:timestamp", and any other types store the + # value under the "@value" key. property_value['@value'] = value['value'] + return property_value def filter_items(regex, pathlist): + """ + Filter a list of file paths to those matching a regex pattern. + + Used by both entrypoint scripts to restrict processing to a subset of + files when the -r / --regex flag is passed on the command line. + + Parameters: + * regex - regex pattern string, compiled with re.compile() + * pathlist - iterable of pathlib.Path or string paths to filter + + Returns a list containing only the paths whose string representation + matches the pattern. + """ reg = re.compile(regex) return [p for p in pathlist if reg.search(str(p))] - -conf_path = get_dir("config/private.yml") -config = get_config(conf_path) -dev_config = get_config(conf_path, "development") -prod_config = get_config(conf_path, "production") - -omeka = OmekaAPIClient(config['omeka_server']) -omeka_auth = OmekaAPIClient( - api_url = config['omeka_server'], - key_identity = config['key_identity'], - key_credential = config['key_credential'] -) -parser = argparse.ArgumentParser() -parser.add_argument('-e', '--environment', required=False, default="development") -parser.add_argument('-m', '--media-skip', action='store_true', - help='Only ingest media not already ingested') -parser.add_argument('-r', '--regex', required=False, help = "Filter files with regex") -args = parser.parse_args() -template_number = config["resource_template"] -omeka_data_base = config["omeka_data_base"] \ No newline at end of file From e32ac2dca413d36038241395c64f407061a4e279 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 16:16:04 -0500 Subject: [PATCH 045/222] remove omeka import; accept omeka_data_base as constructor param --- lib/datura/python/field_definitions.py | 63 +++++++++++++++++++++----- 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index a4c42ff34..f7a816cdd 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -1,10 +1,32 @@ import sys import os from datetime import datetime -import omeka class FieldDefinitions: - #these are the default field definitions, which may be overridden in specific projects + """ + Default field extraction patterns for the Omeka S ingestion pipeline. + + Each method receives the raw JSON item dict (a single record from the + Datura-generated ES output file) and returns the value to be posted for + that Omeka property, or None if the field is absent. + + To override any method for a specific collection, copy + omeka_overrides_example.py to scripts/python/omeka_overrides.py in the + collection repository and subclass FieldDefinitions there. The get_fields() + factory below will load the override class automatically. + """ + + def __init__(self, omeka_data_base=""): + """ + Parameters: + * omeka_data_base - base URL used to construct media URIs in uriData(). + Passed in from OmekaContext.omeka_data_base. + Defaults to "" (empty string) so that instantiation + without arguments is safe in tests. + """ + # Stored as a private attribute and accessed only by uriData(). + self._omeka_data_base = omeka_data_base + def title(self, json): return json.get("title", None) @@ -23,9 +45,12 @@ def category2(self, json): def uriData(self, json): uri_data = json.get("uri_data", None) if uri_data: + # Strip the original path and reconstruct the URI under the + # collection's configured media base URL. The base URL comes + # from the constructor rather than a global so this class can + # be instantiated safely in tests without a live config file. filename = uri_data.split("/")[-1] - omeka_data_base = omeka.omeka_data_base - new_uri_data = f"{omeka_data_base}/{filename}" + new_uri_data = "{}/{}".format(self._omeka_data_base, filename) return new_uri_data def dcterms_type(self, json): @@ -68,9 +93,6 @@ def dcterms_format(self, json): def relation(self, json): relation_ids = [relation['id'] for relation in json.get("has_relation") or [] if 'id' in relation] return relation_ids - - #citation fields - #TODO is citation always single-valued? if array might need to add code to deal with that def publisher(self, json): return (json.get("citation") or {}).get("publisher", None) @@ -217,11 +239,30 @@ def itemText(self, json): text += (" " + self.identifier(json)) return text -def get_fields(): +def get_fields(omeka_data_base=""): + """ + Return the appropriate FieldDefinitions instance for this collection. + + Attempts to import CustomFields from scripts/python/omeka_overrides.py + in the collection directory. If that file does not exist, falls back to + the default FieldDefinitions class. + + Parameters: + * omeka_data_base - passed through to the FieldDefinitions constructor + so that uriData() can build correct media URIs. + Callers should pass ctx.omeka_data_base. + + Returns a FieldDefinitions instance (or a CustomFields subclass of it). + """ try: - #make sure it can override from the right directly + # Insert at position 0 so the collection's scripts/python directory + # takes precedence over any system-installed omeka_overrides module. sys.path.insert(0, './scripts/python') from omeka_overrides import CustomFields - return CustomFields() + # CustomFields inherits __init__ from FieldDefinitions, so + # omeka_data_base is passed through automatically. Override __init__ + # in CustomFields only if you need additional constructor logic. + return CustomFields(omeka_data_base=omeka_data_base) except ImportError: - return FieldDefinitions() \ No newline at end of file + # No collection-specific overrides found; use the defaults. + return FieldDefinitions(omeka_data_base=omeka_data_base) \ No newline at end of file From 17ea5cef0f9c59002ed97391cf2378265df8129a Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 16:24:21 -0500 Subject: [PATCH 046/222] add ctx arg and property ID cache; fix silent exceptions --- lib/datura/python/api_fields.py | 593 ++++++++++++++++++++++---------- 1 file changed, 418 insertions(+), 175 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index c2e55e6c0..27d51466a 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -1,289 +1,532 @@ +""" +api_fields.py + +Transforms a single Datura-generated JSON item into the format expected by the +Omeka S REST API, and resolves inter-item relationships (links) by looking up +CDRH identifiers in the live Omeka instance. + +The two primary entry points called by json_to_omeka.py are: + prepare_item(ctx, row, existing_item) — build or update item metadata + link_records(ctx, row, existing_item) — resolve and attach relationships + +All functions that need API access or configuration now receive a ctx +(OmekaContext) parameter. The property ID cache on ctx (ctx.get_property_id()) +means each Omeka term is looked up only once per run rather than once per field per item. +""" + import json +import logging import re import sys + import omeka from datetime import datetime from field_definitions import get_fields -def build_item_dict(json, existing_item): - """Takes in JSON with CDRH API fields and an existing API item from Omeka S in format. - Returns Omeka API item (in JSON format) updated with values from new CDHR schema for Omeka.""" +# Module-level logger so that log records from this module are identifiable +# by name in the output stream. +logger = logging.getLogger(__name__) + + +def build_item_dict(ctx, json_item, existing_item): + """ + Map a Datura JSON item to an Omeka S item dict, populating all configured + property fields. + + Iterates over the ~70 field definitions in FieldDefinitions (or a + collection-specific CustomFields subclass), extracts each value from the + JSON item, and calls update_item_value() to format and attach it. + + Parameters: + * ctx - OmekaContext providing config and the property ID cache + * json_item - dict representing one record from the Datura ES output + * existing_item - existing Omeka item dict to update in-place, or None + when creating a new item (an empty dict is used instead) + + Returns the built or updated item dict, ready for payload preparation. + Raises ValueError (caught by the caller) if field extraction fails in an + unexpected way. + """ try: - fields = get_fields() + # Load the collection-specific field definitions, falling back to the + # defaults if no omeka_overrides.py exists in scripts/python/. + # Pass omeka_data_base so that uriData() can construct media URIs + # without needing a global. + fields = get_fields(omeka_data_base=ctx.omeka_data_base) + + # Start from the existing Omeka item dict when updating, or an empty + # dict when creating. update_item_value() clears each key before + # writing, so stale values from the existing item are replaced. built_item = existing_item if existing_item else {} - update_item_value(built_item, "dcterms:title", fields.title(json)) - update_item_value(built_item, "dcterms:identifier", fields.identifier(json)) - update_item_value(built_item, "dh:collection", fields.collection(json)) - update_item_value(built_item, "dh:category", fields.category(json)) - update_item_value(built_item, "dh:category2", fields.category2(json)) - update_item_value(built_item, "dh:uriData", fields.uriData(json), "uri") - update_item_value(built_item, "dcterms:type", fields.dcterms_type(json)) - update_item_value(built_item, "dcterms:creator", fields.creator(json)) - update_item_value(built_item, "dcterms:contributor", fields.contributor(json)) - update_item_value(built_item, "dcterms:date", fields.date(json), "numeric:timestamp") - update_item_value(built_item, "dh:dateDisplay", fields.dateDisplay(json)) - update_item_value(built_item, "dh:dateYear", fields.dateYear(json)) - update_item_value(built_item, "dcterms:description", fields.description(json)) - update_item_value(built_item, "dcterms:format", fields.dcterms_format(json)) - update_item_value(built_item, "dcterms:relation", fields.relation(json)) - update_item_value(built_item, "dcterms:publisher", fields.publisher(json)) - update_item_value(built_item, "dh:biblID", fields.biblID(json)) - update_item_value(built_item, "tei:biblTitle", fields.biblTitle(json)) - update_item_value(built_item, "tei:biblPubPlace", fields.biblPubPlace(json)) - update_item_value(built_item, "bibo:issue", fields.issue(json)) - update_item_value(built_item, "bibo:pageStart", fields.pageStart(json)) - update_item_value(built_item, "bibo:pageEnd", fields.pageEnd(json)) - update_item_value(built_item, "bibo:section", fields.section(json)) - update_item_value(built_item, "bibo:volume", fields.volume(json)) - update_item_value(built_item, "tei:biblTitleA", fields.biblTitleA(json)) - update_item_value(built_item, "tei:biblTitleM", fields.biblTitleM(json)) - update_item_value(built_item, "tei:biblTitleJ", fields.biblTitleJ(json)) - update_item_value(built_item, "dcterms:rightsHolder", fields.rightsHolder(json)) - update_item_value(built_item, "dcterms:license", fields.license(json)) - update_item_value(built_item, "dcterms:subject", fields.subject(json)) - update_item_value(built_item, "dh:topic", fields.topic(json)) - update_item_value(built_item, "dh:category3", fields.category3(json)) - update_item_value(built_item, "dh:category4", fields.category4(json)) - update_item_value(built_item, "dh:category5", fields.category5(json)) - update_item_value(built_item, "dh:note", fields.note(json)) - update_item_value(built_item, "dcterms:abstract", fields.abstract(json)) - update_item_value(built_item, "dh:keyword", fields.keyword(json)) - update_item_value(built_item, "dh:keyword2", fields.keyword2(json)) - update_item_value(built_item, "dh:keyword3", fields.keyword3(json)) - update_item_value(built_item, "dh:keyword4", fields.keyword4(json)) - update_item_value(built_item, "dh:keyword5", fields.keyword5(json)) - update_item_value(built_item, "dcterms:source", fields.source(json)) - update_item_value(built_item, "dcterms:medium", fields.medium(json)) - update_item_value(built_item, "dcterms:extent", fields.extent(json)) - update_item_value(built_item, "dcterms:language", fields.language(json)) - update_item_value(built_item, "dh:box", fields.box(json)) - update_item_value(built_item, "dh:folder", fields.folder(json)) - update_item_value(built_item, "foaf:name", fields.name(json)) - update_item_value(built_item, "dh:spatial_short_name", fields.spatial_short_name(json)) - update_item_value(built_item, "tei:correspSentName", fields.correspSentName(json)) - update_item_value(built_item, "tei:correspSentPlace", fields.correspSentPlace(json)) - update_item_value(built_item, "tei:correspSentDate", fields.correspSentDate(json), "numeric:timestamp") - update_item_value(built_item, "tei:correspDeliveredName", fields.correspDeliveredName(json)) - update_item_value(built_item, "tei:correspDeliveredPlace", fields.correspDeliveredPlace(json)) - update_item_value(built_item, "tei:correspDeliveredDate", fields.correspDeliveredDate(json), "numeric:timestamp") - update_item_value(built_item, "tei:distributor", fields.distributor(json)) - update_item_value(built_item, "tei:authority", fields.authority(json)) - update_item_value(built_item, "tei:biblNote", fields.biblNote(json)) - update_item_value(built_item, "dh:annotationsText", fields.annotationsText(json)) - update_item_value(built_item, "dh:itemText", fields.itemText(json)) + + update_item_value(ctx, built_item, "dcterms:title", fields.title(json_item)) + update_item_value(ctx, built_item, "dcterms:identifier", fields.identifier(json_item)) + update_item_value(ctx, built_item, "dh:collection", fields.collection(json_item)) + update_item_value(ctx, built_item, "dh:category", fields.category(json_item)) + update_item_value(ctx, built_item, "dh:category2", fields.category2(json_item)) + update_item_value(ctx, built_item, "dh:uriData", fields.uriData(json_item), "uri") + update_item_value(ctx, built_item, "dcterms:type", fields.dcterms_type(json_item)) + update_item_value(ctx, built_item, "dcterms:creator", fields.creator(json_item)) + update_item_value(ctx, built_item, "dcterms:contributor", fields.contributor(json_item)) + update_item_value(ctx, built_item, "dcterms:date", fields.date(json_item), "numeric:timestamp") + update_item_value(ctx, built_item, "dh:dateDisplay", fields.dateDisplay(json_item)) + update_item_value(ctx, built_item, "dh:dateYear", fields.dateYear(json_item)) + update_item_value(ctx, built_item, "dcterms:description", fields.description(json_item)) + update_item_value(ctx, built_item, "dcterms:format", fields.dcterms_format(json_item)) + update_item_value(ctx, built_item, "dcterms:relation", fields.relation(json_item)) + update_item_value(ctx, built_item, "dcterms:publisher", fields.publisher(json_item)) + update_item_value(ctx, built_item, "dh:biblID", fields.biblID(json_item)) + update_item_value(ctx, built_item, "tei:biblTitle", fields.biblTitle(json_item)) + update_item_value(ctx, built_item, "tei:biblPubPlace", fields.biblPubPlace(json_item)) + update_item_value(ctx, built_item, "bibo:issue", fields.issue(json_item)) + update_item_value(ctx, built_item, "bibo:pageStart", fields.pageStart(json_item)) + update_item_value(ctx, built_item, "bibo:pageEnd", fields.pageEnd(json_item)) + update_item_value(ctx, built_item, "bibo:section", fields.section(json_item)) + update_item_value(ctx, built_item, "bibo:volume", fields.volume(json_item)) + update_item_value(ctx, built_item, "tei:biblTitleA", fields.biblTitleA(json_item)) + update_item_value(ctx, built_item, "tei:biblTitleM", fields.biblTitleM(json_item)) + update_item_value(ctx, built_item, "tei:biblTitleJ", fields.biblTitleJ(json_item)) + update_item_value(ctx, built_item, "dcterms:rightsHolder", fields.rightsHolder(json_item)) + update_item_value(ctx, built_item, "dcterms:license", fields.license(json_item)) + update_item_value(ctx, built_item, "dcterms:subject", fields.subject(json_item)) + update_item_value(ctx, built_item, "dh:topic", fields.topic(json_item)) + update_item_value(ctx, built_item, "dh:category3", fields.category3(json_item)) + update_item_value(ctx, built_item, "dh:category4", fields.category4(json_item)) + update_item_value(ctx, built_item, "dh:category5", fields.category5(json_item)) + update_item_value(ctx, built_item, "dh:note", fields.note(json_item)) + update_item_value(ctx, built_item, "dcterms:abstract", fields.abstract(json_item)) + update_item_value(ctx, built_item, "dh:keyword", fields.keyword(json_item)) + update_item_value(ctx, built_item, "dh:keyword2", fields.keyword2(json_item)) + update_item_value(ctx, built_item, "dh:keyword3", fields.keyword3(json_item)) + update_item_value(ctx, built_item, "dh:keyword4", fields.keyword4(json_item)) + update_item_value(ctx, built_item, "dh:keyword5", fields.keyword5(json_item)) + update_item_value(ctx, built_item, "dcterms:source", fields.source(json_item)) + update_item_value(ctx, built_item, "dcterms:medium", fields.medium(json_item)) + update_item_value(ctx, built_item, "dcterms:extent", fields.extent(json_item)) + update_item_value(ctx, built_item, "dcterms:language", fields.language(json_item)) + update_item_value(ctx, built_item, "dh:box", fields.box(json_item)) + update_item_value(ctx, built_item, "dh:folder", fields.folder(json_item)) + update_item_value(ctx, built_item, "foaf:name", fields.name(json_item)) + update_item_value(ctx, built_item, "dh:spatial_short_name", fields.spatial_short_name(json_item)) + update_item_value(ctx, built_item, "tei:correspSentName", fields.correspSentName(json_item)) + update_item_value(ctx, built_item, "tei:correspSentPlace", fields.correspSentPlace(json_item)) + update_item_value(ctx, built_item, "tei:correspSentDate", fields.correspSentDate(json_item), "numeric:timestamp") + update_item_value(ctx, built_item, "tei:correspDeliveredName", fields.correspDeliveredName(json_item)) + update_item_value(ctx, built_item, "tei:correspDeliveredPlace", fields.correspDeliveredPlace(json_item)) + update_item_value(ctx, built_item, "tei:correspDeliveredDate", fields.correspDeliveredDate(json_item), "numeric:timestamp") + update_item_value(ctx, built_item, "tei:distributor", fields.distributor(json_item)) + update_item_value(ctx, built_item, "tei:authority", fields.authority(json_item)) + update_item_value(ctx, built_item, "tei:biblNote", fields.biblNote(json_item)) + update_item_value(ctx, built_item, "dh:annotationsText", fields.annotationsText(json_item)) + update_item_value(ctx, built_item, "dh:itemText", fields.itemText(json_item)) + return built_item + except ValueError as e: - #breakpoint() - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - + # A ValueError here means a field definition returned an unexpected + # type or structure. Log it and re-raise so the caller can record the + # error and skip this item. + logger.error("ValueError building item dict: %s", e) + raise + + +def link_item(ctx, json_item, existing_item): + """ + Resolve inter-item relationships for a single item and attach them to the + existing Omeka item dict. + + Each relationship type (has_part, is_part_of, etc.) is handled in its own + try/except block. A missing or None field in the JSON is expected for most + items — these are caught as KeyError or TypeError and logged at DEBUG level + so they do not pollute the run log. Genuine API failures in + link_item_record() will surface as exceptions and should be caught by the + caller (link_item in json_to_omeka.py). -#TODO change item linking for JSON and new API -def link_item(json_item, existing_item): + Parameters: + * ctx - OmekaContext providing the API client and item_set_id + * json_item - raw JSON item dict from the Datura ES output + * existing_item - the current Omeka item dict, deepcopied by the caller + + Returns the updated existing_item dict with relationship fields populated. + """ + # Each relationship field is optional; most items will not have all of + # them. Missing fields generate a DEBUG log entry, not a warning. - #has_part try: part_ids = [part['id'] for part in json_item["has_part"]] - link_item_record(existing_item, "dcterms:hasPart", part_ids) - except Exception: - pass - #is_part_of + link_item_record(ctx, existing_item, "dcterms:hasPart", part_ids) + except (KeyError, TypeError) as e: + logger.debug("No has_part data for %s: %s", json_item.get("identifier"), e) + try: - link_item_record(existing_item, "dcterms:isPartOf", json_item["is_part_of"]["id"]) - except Exception: - pass - #has_relation + link_item_record(ctx, existing_item, "dcterms:isPartOf", json_item["is_part_of"]["id"]) + except (KeyError, TypeError) as e: + logger.debug("No is_part_of data for %s: %s", json_item.get("identifier"), e) + try: - link_item_record(existing_item, "dcterms:relation", json_item["has_relation"]["id"]) - except Exception: - pass - #previous + link_item_record(ctx, existing_item, "dcterms:relation", json_item["has_relation"]["id"]) + except (KeyError, TypeError) as e: + logger.debug("No has_relation data for %s: %s", json_item.get("identifier"), e) + try: - link_item_record(existing_item, "dh:orderPrev", json_item["previous_item"]["id"]) - except Exception: - pass - #next + link_item_record(ctx, existing_item, "dh:orderPrev", json_item["previous_item"]["id"]) + except (KeyError, TypeError) as e: + logger.debug("No previous_item data for %s: %s", json_item.get("identifier"), e) + try: - link_item_record(existing_item, "dh:orderNext", json_item["next_item"]["id"]) - except Exception: - pass + link_item_record(ctx, existing_item, "dh:orderNext", json_item["next_item"]["id"]) + except (KeyError, TypeError) as e: + logger.debug("No next_item data for %s: %s", json_item.get("identifier"), e) + try: - link_item_record(existing_item, "tei:correspNext", json_item["correspNext_omeka_s"]) - except Exception: - pass + link_item_record(ctx, existing_item, "tei:correspNext", json_item["correspNext_omeka_s"]) + except (KeyError, TypeError) as e: + logger.debug("No correspNext_omeka_s data for %s: %s", json_item.get("identifier"), e) + try: - link_item_record(existing_item, "tei:correspPrev", json_item["correspPrev_omeka_s"]) - except Exception: - pass + link_item_record(ctx, existing_item, "tei:correspPrev", json_item["correspPrev_omeka_s"]) + except (KeyError, TypeError) as e: + logger.debug("No correspPrev_omeka_s data for %s: %s", json_item.get("identifier"), e) + return existing_item -def prepare_item(row, existing_item = None): - item_dict = build_item_dict(row, existing_item) - # TODO add conditional logic for different templates? - return item_dict -def link_records(row, existing_item): - item_dict = link_item(row, existing_item) - # TODO add conditional logic? - return item_dict +def prepare_item(ctx, row, existing_item=None): + """ + Build a complete Omeka item dict from a Datura JSON record. + + Thin wrapper around build_item_dict() that provides the standard entry + point used by json_to_omeka.py for both new item creation and updates. + + Parameters: + * ctx - OmekaContext + * row - raw JSON item dict from the Datura ES output + * existing_item - existing Omeka item dict when updating, or None when + creating a new item + + Returns the built item dict, or raises ValueError if field extraction fails. + """ + # TODO: add conditional logic here for items that need a different template + return build_item_dict(ctx, row, existing_item) + + +def link_records(ctx, row, existing_item): + """ + Resolve and attach all relationship fields for a single item. + + Thin wrapper around link_item() that provides the standard entry point + used by json_to_omeka.py during the linking pass. + + Parameters: + * ctx - OmekaContext + * row - raw JSON item dict from the Datura ES output + * existing_item - the current Omeka item dict (deepcopied by the caller) + + Returns the updated item dict. + """ + # TODO: add conditional logic here if different relationship schemas are needed + return link_item(ctx, row, existing_item) + def get_json_value(row, name): + """ + Extract a value from a CSV-derived row dict, handling multiple encodings. + + Datura serialises multi-valued fields from CSV in two ways: + - JSON array strings: '["value1", "value2"]' + - Semicolon-delimited strings: 'value1;;;value2' + + Single values are returned as-is. Empty strings return the empty string. + + Parameters: + * row - dict representing one CSV row + * name - the field name to extract + + Returns a string, list of strings, or empty string. + """ if len(row[name]) > 0: if row[name].startswith('["'): + # Deserialise a JSON-encoded array. return json.loads(row[name]) elif ";;;" in row[name]: + # Split a semicolon-delimited multi-value string. return row[name].split(";;;") else: return row[name] else: return row[name] - -def update_item_value(item, key, value, datatype="literal"): + + +def update_item_value(ctx, item, key, value, datatype="literal"): """ - takes in JSON representation of API item, the field name, the value to add or update, and a datatype (defaults to "literal") - value may be in string format or list. Should be able to modify existing values and update new ones. (Note that there are still issues with updating fields - returns the JSON hash with the updated value + Set or replace a property on an Omeka item dict. + + Clears the existing value list for the key (if any) and writes the new + value(s). This ensures that re-running the ingest for an existing item + replaces stale values rather than appending duplicates. + + If value is None or an empty list, the key is initialised to [] and no + formatted values are added — this effectively clears the field in Omeka + when the item is PUT back. + + Parameters: + * ctx - OmekaContext (passed through to add_formatted_value) + * item - the Omeka item dict being built + * key - Omeka property term string, e.g. "dcterms:title" + * value - the value to set; may be a string, int, float, or list. + None and empty list are treated as "no value". + * datatype - Omeka data type string (default "literal"). Use + "uri" for URLs or "numeric:timestamp" for dates. """ - #clear the existing values of the key, or initialize it if it is new + # Always reset the key so that old values from a prior ingest are not + # carried forward when the source data no longer has a value for this field. item[key] = [] + if type(value) in [str, int, float]: - item = add_formatted_value(item, key, value, datatype) + item = add_formatted_value(ctx, item, key, value, datatype) elif type(value) == list: - # make sure values are unique + # Deduplicate and remove None entries before iterating. value = list(set(value)) - value = [v for v in value if v is not None] # remove None values from values + value = [v for v in value if v is not None] for v in value: - item = add_formatted_value(item, key, v, datatype) - -def add_formatted_value(item, key, value, datatype, label=""): - # takes in item, key, value, and datatype, returns item with key set or added to value, and formatted in the format Omeka S - # expects as indicated in the template - # used when adding a new value that is not already in the Omeka JSON, so that Omeka will properly update the value - # this comes up - # literal values should be str + item = add_formatted_value(ctx, item, key, v, datatype) + + +def add_formatted_value(ctx, item, key, value, datatype, label=""): + """ + Format a single value and append it to the property list on an item dict. + + Calls ctx.get_property_id() to obtain the numeric Omeka property ID for + the given term. The cache on ctx ensures this API call is made at most once + per unique term per run. + + Parameters: + * ctx - OmekaContext; provides get_property_id() and the API client + * item - the Omeka item dict being built + * key - Omeka property term string, e.g. "dcterms:title" + * value - the scalar value to format + * datatype - Omeka data type string, e.g. "literal", "uri", + "numeric:timestamp" + * label - optional display label for URI values + + Returns the item dict with the new value appended to item[key]. + """ + # Coerce to string for literal values to avoid sending a bare int or float + # in the API payload, which Omeka S may reject. if datatype == "literal": value = str(value) - prop_id = omeka.omeka_auth.get_property_id(key) + + # Look up the property ID via the cache — avoids one API round-trip per + # field per item across the entire run. + prop_id = ctx.get_property_id(key) + prop_value = { "value": value, - "type": datatype + "type": datatype, } + # Use the custom prepare_property_value from omeka.py, which supports the + # label parameter for URI types. For resource:item links, use + # ctx.client.prepare_property_value() instead (see link_item_record). formatted = omeka.prepare_property_value(prop_value, prop_id, label) + if key in item and type(item[key]) == list: item[key].append(formatted) else: item[key] = [formatted] + return item + def get_matching_ids_from_markdown(row, field): - # takes in an array of strings in markdown format, which include CDRH IDs - # returns an array of just the IDs + """ + Extract CDRH identifier strings from a field containing markdown-formatted links. + + Markdown link format: [Display Name](identifier) + This function extracts only the identifier (the part in parentheses). + + Parameters: + * row - dict representing one Datura JSON item + * field - the field name containing markdown link strings + + Returns a list of identifier strings, or an empty list if the field is + absent or contains no valid links. + """ if row[field]: markdown_values = sorted(get_json_value(row, field)) ids = [] if markdown_values: - #should be either single value or array if type(markdown_values) == str: match = re.search(r"\]\((.*)\)", markdown_values) if match: - id_no = match.group(1) - ids.append(id_no) + ids.append(match.group(1)) else: for value in markdown_values: - #parse with regex to get ids match = re.search(r"\]\((.*)\)", value) if match: - id_no = match.group(1) - ids.append(id_no) + ids.append(match.group(1)) if len(ids) > 1: + # Remove empty strings that may result from links with no + # destination (e.g. "[Name]()"). ids = list(filter(None, ids)) - return ids - + return ids else: return [] - + + def get_matching_names_from_markdown(row, field): - # takes in an array of strings in markdown format, which include names - # returns an array of just the names - # filters out the ones that have a corresponding id, it is not necessary to get their names + """ + Extract display names from a field containing markdown-formatted links, + filtering out names that have a corresponding identifier. + + Markdown link format: [Display Name](identifier) + This function extracts only the display name (the part in brackets), but + skips entries where an identifier is also present, since those items can + be resolved by ID via get_matching_ids_from_markdown. + + Parameters: + * row - dict representing one Datura JSON item + * field - the field name containing markdown link strings + + Returns a list of display name strings, or an empty list if the field is + absent or all entries have identifiers. + """ if row[field]: markdown_values = get_json_value(row, field) names = [] - if markdown_values: - #should be either single value or array if type(markdown_values) == str: name_match = re.search(r"\[(.*?)\]", markdown_values) - # filter out entries that have ids id_match = re.search(r"\]\((.*)\)", markdown_values) + # Only collect the name if there is no associated identifier. if name_match and not id_match.group(1): - name = name_match.group(1) - names.append(name) + names.append(name_match.group(1)) else: for value in markdown_values: - #parse with regex to get ids name_match = re.search(r"\[(.*?)\]", value) id_match = re.search(r"\]\((.*)\)", value) if name_match and not id_match.group(1): - name = name_match.group(1) - names.append(name) - return names + names.append(name_match.group(1)) + return names else: return [] -def get_omeka_ids(lookup_values, filter_property, item_set_id = None): - item_set_id = omeka.get_item_set() + +def get_omeka_ids(ctx, lookup_values, filter_property): + """ + Resolve a list of lookup values to Omeka numeric item IDs. + + For each lookup value, queries the Omeka API to find the matching item + within the configured item set. Used during the linking pass to convert + CDRH identifiers into the Omeka IDs required for resource:item links. + + Parameters: + * ctx - OmekaContext providing the API client and item_set_id + * lookup_values - a single value or list of values to look up; typically + CDRH identifier strings but may be Omeka IDs directly + when filter_property is "o:id" + * filter_property - the Omeka property to match against, e.g. + "dcterms:identifier" or "o:id" + + Returns a list of integer Omeka item IDs for all successfully resolved values. + Logs a warning for values that cannot be resolved. + """ omeka_ids = [] - #lookup_values are usually a list of cdrh_ids, but may be another value + + # Normalise a single value to a list for uniform iteration. lookup_values = [lookup_values] if not isinstance(lookup_values, list) else lookup_values + for lookup_value in lookup_values: + # Skip blank or None values — these are common when optional relation + # fields are absent in some records but not others. if not lookup_value or lookup_value == '': continue + if filter_property == "o:id": + # The value is already an Omeka ID; cast to int and add directly. omeka_ids.append(int(lookup_value)) else: - match = omeka.omeka_auth.filter_items_by_property(filter_property = filter_property, filter_value = lookup_value, item_set_id=item_set_id) + match = ctx.client.filter_items_by_property( + filter_property=filter_property, + filter_value=lookup_value, + item_set_id=ctx.item_set_id, + ) if match["total_results"] >= 1: if match["total_results"] > 1: - print(f"warning: multiple matches for {lookup_value}, taking first match") - #breakpoint() - omeka_id = match['results'][0]["o:id"] - omeka_ids.append(omeka_id) + # Multiple matches indicate a data integrity issue; take + # the first result and log a warning for investigation. + logger.warning( + "Multiple matches for %r, taking first result", lookup_value + ) + omeka_ids.append(match['results'][0]["o:id"]) else: - print(f"Unable to link {lookup_value}, no matches") + logger.warning("Unable to link %r: no matching items found", lookup_value) + return omeka_ids +def link_item_record(ctx, item, key, values, item_set=False, filter_property="dcterms:identifier"): + """ + Resolve lookup values to Omeka IDs and attach them as resource links on + the item dict. + + Clears the existing value list for the key before writing, so re-running + this function replaces stale links rather than appending duplicates. + + Parameters: + * ctx - OmekaContext providing the API client + * item - the Omeka item dict being built + * key - Omeka property term for this relationship, + e.g. "dcterms:hasPart" or "dh:orderNext" + * values - lookup values to resolve; either already-resolved Omeka + IDs (when item_set=True) or CDRH identifiers to look up + * item_set - if True, treat values as Omeka item set IDs rather than + item IDs; uses "resource:itemset" type and adds the + extra fields required by the item-sets plugin + * filter_property - the Omeka property to use when looking up items by value; + defaults to "dcterms:identifier" -def link_item_record(item, key, values, item_set=False, filter_property = "dcterms:identifier"): - omeka_ids = values if item_set else get_omeka_ids(values, filter_property) - #dedupe + Returns the updated item dict. + """ + # When item_set=True the caller has already resolved the IDs; otherwise + # resolve them from CDRH identifiers via the API. + omeka_ids = values if item_set else get_omeka_ids(ctx, values, filter_property) + + # Deduplicate while preserving order (dict.fromkeys is stable in Python 3.7+). omeka_ids = list(dict.fromkeys(omeka_ids)) - prop_id = omeka.omeka_auth.get_property_id(key) - #always clear items + + # Look up the property ID via the cache. + prop_id = ctx.get_property_id(key) + + # Always clear the existing values for this relationship field so that + # stale links from a prior ingest are removed. item[key] = [] resource_type = "resource:itemset" if item_set else "resource:item" + for omeka_id in omeka_ids: - #make sure item isn't already linked, to avoid duplicates - if not item[key] or not omeka_id in [value.get("value_resource_id") for value in item[key]]: + # Guard against duplicate links — check whether this Omeka ID is + # already present in the list before appending. + already_linked = ( + item[key] and + omeka_id in [v.get("value_resource_id") for v in item[key]] + ) + if not already_linked: prop_value = { "type": resource_type, - "value": omeka_id + "value": omeka_id, } - formatted = omeka.omeka_auth.prepare_property_value(prop_value, prop_id) - #different format for item sets, plugin doesn't do it automatically + # Use the library's prepare_property_value (via ctx.client) for + # resource links, not the custom omeka.py version — the library + # version correctly handles the value_resource_id field. + formatted = ctx.client.prepare_property_value(prop_value, prop_id) + if item_set: - formatted['@id'] = f'{omeka.omeka_auth.api_url}/item_sets/{omeka_id}' + # The item-sets plugin requires these extra fields in addition + # to what prepare_property_value generates. + formatted['@id'] = '{}/item_sets/{}'.format(ctx.client.api_url, omeka_id) formatted['value_resource_id'] = omeka_id - formatted["value_resource_name"] = "item_sets" + formatted['value_resource_name'] = 'item_sets' + item[key].append(formatted) - return item -def build_citation(row): - # TODO format the date better - if row["publisher"]: - return f""" - "{row["title"]}", {json.loads(row["publisher"])[0]}, {row["Article Date (formatted)"]}, {row["Source page no"]}. - Accessed {row["Source access date"]}. {row["Source link"]}. - """ + return item From f925bcc6b714ede0f5feb714b740527ee71558af Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 16:36:07 -0500 Subject: [PATCH 047/222] wrap body in main(); fix hardcoded path to use env; internal functions receive ctx; use logger --- lib/datura/python/json_to_omeka.py | 536 +++++++++++++++++++++++------ 1 file changed, 432 insertions(+), 104 deletions(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index bf572643e..8fb366806 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -1,122 +1,450 @@ -#import necessary modules, including omeka, and the fields -import omeka -import api_fields +""" +json_to_omeka.py + +Entrypoint script: reads Datura-generated Elasticsearch JSON files and posts +each item to an Omeka S instance. + +The script runs in two sequential passes: + + Pass 1 — Item posting (post_items): + For each JSON item, check whether the Omeka identifier already exists. + * If found (one result): update the existing Omeka item's metadata. + * If not found: create a new Omeka item. + * If multiple results: warn and skip (data integrity problem). + + Pass 2 — Item linking (link_items): + After all items exist in Omeka, re-read the same JSON files and populate + relational fields (has_part, has_source, etc.) by resolving Omeka IDs for + the referenced items. A separate pass is required because items must exist + before they can be linked to one another. +Usage (from collection root directory): + python3 json_to_omeka.py -e development + python3 json_to_omeka.py -e production -r "some_pattern" + python3 json_to_omeka.py -e development --log-level DEBUG + +The script is invoked by bin/post_omeka in the Datura gem. The Ruby wrapper +passes -e and -r arguments from its own CLI. No changes to bin/post_omeka +are required to support the --log-level flag (it defaults to INFO). +""" + +import argparse import copy import json -from pathlib import Path -#needed for debugging purposes and path +import logging +import os import sys import traceback -import os +from pathlib import Path + +import api_fields +import omeka +from omeka_context import ( + OmekaAPIError, + OmekaContext, + OmekaItemNotFoundError, + OmekaMultipleMatchesError, + configure_logging, +) +from omeka import filter_items, prepare_item_payload_using_template + +# Module-level logger. Records from this module appear as "json_to_omeka" +# in log output so they can be filtered independently from other modules. +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# CLI argument parsing +# --------------------------------------------------------------------------- + +def _parse_args(): + """ + Parse command-line arguments for the JSON-to-Omeka entrypoint. + + Returns an argparse.Namespace with: + * environment - "development" or "production" + * regex - optional file-filter pattern string, or None + * log_level - logging level string, default "INFO" + + Note: this entrypoint has no --media-skip flag. That flag belongs only + to html_and_media_ingest.py, which handles media re-ingestion. + getattr(args, "media_skip", False) in OmekaContext.from_args() handles + its absence gracefully. + """ + parser = argparse.ArgumentParser( + description="Post Datura ES JSON output to an Omeka S instance." + ) + parser.add_argument( + "-e", "--environment", + required=True, + help="Target environment: 'development' or 'production'.", + ) + parser.add_argument( + "-r", "--regex", + default=None, + help=( + "Optional regex pattern to restrict processing to matching " + "file paths. Example: -r 'abc123' processes only files whose " + "path contains 'abc123'." + ), + ) + parser.add_argument( + "--log-level", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + dest="log_level", + help="Set the logging verbosity (default: INFO).", + ) + return parser.parse_args() + + +# --------------------------------------------------------------------------- +# Pass 1: item creation / update +# --------------------------------------------------------------------------- -def post_items(pathlist): - #iterate through each file +def post_items(ctx, pathlist): + """ + First pass: create or update Omeka items for every JSON record. + + Iterates over all JSON files in pathlist. For each record: + * Items whose identifier already exists in Omeka are updated in place. + * Items not yet in Omeka are created from scratch. + * Items that return multiple Omeka matches are skipped with a warning + (duplicate identifiers indicate a data integrity problem that must be + resolved in the Omeka admin UI before the item can be re-posted). + * Items whose identifier field is falsy are skipped silently. + + Per-item errors (API failures, malformed payload) are recorded via + ctx.record_error() and do NOT halt the run. Fatal errors (wrong + credentials, missing config) raise exceptions that propagate to main(). + + Parameters: + * ctx - OmekaContext providing API client, config, and error log + * pathlist - list of pathlib.Path objects pointing to ES JSON files + """ for path in pathlist: filename = str(path) with open(filename) as jsonfile: json_items = json.load(jsonfile) - # TODO change template_number to actual number, account for other schemas is necessary - template_number = omeka.template_number - for json_item in json_items: - try: - if not json_item["identifier"]: - #breakpoint() - print("skipping item without identifier") - continue - except TypeError as e: - #breakpoint() - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - - matching_items = omeka.omeka_auth.filter_items_by_property(filter_property = "dcterms:identifier", filter_value = json_item["identifier"], item_set_id=item_set_id) - if matching_items: - #if item exists, update item - if matching_items["total_results"] == 1: - update_existing_item(json_item, matching_items) - #add new item if item does not exist - elif matching_items["total_results"] == 0: - add_new_item(json_item, template_number) - #if multiple matches, warn but don't ingest - else: - print(f"multiple matches for {json_item['identifier']}, please check Omeka admin site") - -def link_items(pathlist): - #go through tables again to link records + + # template_number is stable for the lifetime of a run — read from + # ctx rather than re-reading from config for every item. + template_number = ctx.template_number + + for json_item in json_items: + identifier = json_item.get("identifier") + if not identifier: + # Records without an identifier cannot be matched or created. + # This is expected for some document types; log at DEBUG only. + logger.debug("Skipping item without identifier in %s", filename) + continue + + try: + matching_items = ctx.client.filter_items_by_property( + filter_property="dcterms:identifier", + filter_value=identifier, + item_set_id=ctx.item_set_id, + ) + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "filter_items", err)) + continue + + if not matching_items: + # API returned no response object at all — treat as a lookup + # failure rather than "zero results". + logger.warning( + "Unexpected empty response from filter_items for %r; skipping", + identifier, + ) + continue + + total = matching_items.get("total_results", 0) + + if total == 1: + # Item exists — refresh its metadata. + update_existing_item(ctx, json_item, matching_items) + elif total == 0: + # Item is new — create it. + add_new_item(ctx, json_item, template_number) + else: + # More than one match — cannot determine which to update. + logger.warning( + "Multiple matches (%d) for %r; check Omeka admin site", + total, + identifier, + ) + + +def add_new_item(ctx, json_item, template_number): + """ + Build the Omeka payload for a new item and POST it to the API. + + Calls api_fields.prepare_item() to extract and format each field from + the Datura JSON record. If preparation produces no payload (e.g. all + fields were absent), the item is skipped with a warning rather than + POSTing an empty object. + + Parameters: + * ctx - OmekaContext + * json_item - dict: one record from a Datura ES JSON file + * template_number - Omeka resource template numeric ID (from ctx.template_number) + """ + identifier = json_item.get("identifier", "unknown") + + new_item = api_fields.prepare_item(ctx, json_item) + if not new_item: + logger.warning("Could not prepare payload for %r; skipping", identifier) + return + + # Log the identifier we are about to create. Use .get() with a default + # rather than direct dict access so a missing dcterms:identifier key + # does not raise KeyError and halt the run. + title_val = ( + new_item.get("dcterms:identifier", [{}])[0].get("@value", identifier) + ) + logger.info("Creating item %r", title_val) + + # Validate terms against the resource template and wrap values in the + # JSON-LD structure Omeka S expects. + payload = prepare_item_payload_using_template(ctx, new_item, template_number) + + try: + ctx.client.add_item( + payload, + template_id=template_number, + item_set_id=ctx.item_set_id, + is_public=ctx.is_public, + ) + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "add_item", err)) + + +def update_existing_item(ctx, json_item, matching_items): + """ + Re-prepare an existing Omeka item's metadata and PATCH it via the API. + + Fetches the current Omeka item, merges it with fresh values from the + Datura JSON record, and calls update_resource() to apply the changes. + + Parameters: + * ctx - OmekaContext + * json_item - dict: one record from a Datura ES JSON file + * matching_items - dict: Omeka API response containing the existing item + under matching_items["results"][0] + """ + identifier = json_item.get("identifier", "unknown") + + # Log the Omeka-side identifier to make it easy to correlate log lines + # with records in the Omeka admin UI. + omeka_id_display = ( + matching_items["results"][0] + .get("dcterms:identifier", [{}])[0] + .get("@value", identifier) + ) + logger.info("Updating item %r", omeka_id_display) + + # Deep-copy to avoid mutating the dict returned by the API client; the + # original might be referenced elsewhere (e.g. in link_items). + item_to_update = copy.deepcopy(matching_items["results"][0]) + updated_item = api_fields.prepare_item(ctx, json_item, item_to_update) + if not updated_item: + logger.warning("Could not prepare update payload for %r; skipping", identifier) + return + + try: + ctx.client.update_resource(updated_item, "items") + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "update_resource", err)) + + +# --------------------------------------------------------------------------- +# Pass 2: record linking +# --------------------------------------------------------------------------- + +def link_items(ctx, pathlist): + """ + Second pass: populate relational fields between Omeka items. + + Re-reads the same JSON files processed in pass 1. For each record, + resolves the Omeka item IDs of related items (has_part, has_source, etc.) + and PATCHes the item with link values. + + A separate pass is necessary because linked items must already exist in + Omeka before they can be referenced. Running this pass after all items + have been created (or updated) in pass 1 guarantees that the target items + are present. + + Items that cannot be found or that have multiple matches are skipped with + a warning; they will be logged in the run summary. + + Parameters: + * ctx - OmekaContext + * pathlist - list of pathlib.Path objects (same list used in post_items) + """ for path in pathlist: filename = str(path) with open(filename) as jsonfile: json_items = json.load(jsonfile) - for json_item in json_items: - if not json_item["identifier"]: - print("skipping item without identifier") - continue - matching_items = omeka.omeka_auth.filter_items_by_property(filter_property = "dcterms:identifier", filter_value = json_item["identifier"], item_set_id=item_set_id) - if matching_items and matching_items["total_results"] == 1: - #if item exists, update item with linked records - link_item(json_item, matching_items) - else: - #if multiple matches or item not found, display warning - print(f"skipping {json_item['identifier']}, item not properly ingested") - -def link_item(json_item, matching_items): - item_id = matching_items["results"][0]["dcterms:identifier"][0]["@value"] - print(f"linking records for {item_id}") + + for json_item in json_items: + identifier = json_item.get("identifier") + if not identifier: + logger.debug("Skipping item without identifier in %s", filename) + continue + + try: + matching_items = ctx.client.filter_items_by_property( + filter_property="dcterms:identifier", + filter_value=identifier, + item_set_id=ctx.item_set_id, + ) + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "filter_items (link pass)", err)) + continue + + if not matching_items: + logger.warning( + "Unexpected empty response from filter_items for %r during link pass; skipping", + identifier, + ) + continue + + total = matching_items.get("total_results", 0) + if total == 1: + _link_item(ctx, json_item, matching_items) + else: + # total == 0: item was not successfully posted in pass 1. + # total > 1: data integrity problem (duplicate identifiers). + # In both cases, linking is impossible. + logger.warning( + "Skipping link pass for %r: expected 1 match, got %d", + identifier, + total, + ) + + +def _link_item(ctx, json_item, matching_items): + """ + Resolve and attach relational fields for a single Omeka item. + + Extracts the Omeka item from matching_items, calls api_fields.link_records() + to populate relation fields, and PATCHes the result back to the API. + + Named with a leading underscore to signal that it is an internal helper + called only by link_items(); external code should use link_items(). + + Parameters: + * ctx - OmekaContext + * json_item - dict: one record from a Datura ES JSON file + * matching_items - dict: Omeka API response containing the item to link + """ + # Pull the human-readable identifier from the Omeka item for log messages. + item_id = ( + matching_items["results"][0] + .get("dcterms:identifier", [{}])[0] + .get("@value", json_item.get("identifier", "unknown")) + ) + logger.info("Linking records for %r", item_id) + + # Deep-copy so that api_fields.link_records() can modify the item dict + # without affecting the in-memory copy used elsewhere in this pass. item_to_link = copy.deepcopy(matching_items["results"][0]) - linked_item = api_fields.link_records(json_item, item_to_link) + try: - omeka.omeka_auth.update_resource(linked_item, "items") + linked_item = api_fields.link_records(ctx, json_item, item_to_link) except Exception as err: - print(str(err)) - traceback.print_exc() - print(f"Error updating item {item_id}") - #breakpoint() - pass - -def add_new_item(json_item, template_number): - new_item = api_fields.prepare_item(json_item) - if new_item: - try: - print(f"creating item {new_item['dcterms:identifier'][0]['@value']}") - except KeyError as err: - print(err) - #breakpoint() - sys.exit(1) - payload = omeka.prepare_item_payload_using_template(new_item, template_number) - # add item set - try: - omeka.omeka_auth.add_item(payload, template_id=template_number, item_set_id=item_set_id, is_public=is_public) - except Exception as err: - print(err) - #breakpoint() - sys.exit(1) - else: - print(f"error preparing item {json_item['identifier']}") - -def update_existing_item(json_item, matching_items): - print(f"updating item {matching_items['results'][0]['dcterms:identifier'][0]['@value']}") - item_to_update = copy.deepcopy(matching_items["results"][0]) - updated_item = api_fields.prepare_item(json_item, item_to_update) - if updated_item: - try: - omeka.omeka_auth.update_resource(updated_item, "items") - except Exception as err: - print(err) - #breakpoint() - sys.exit(1) - -#look for the output folder: /output/development/es and get all items -json_dir = omeka.get_dir("output/development/es") -regex = omeka.get_regex() -pathlist = list(Path(json_dir).glob('**/*.json')) -if regex: - pathlist = omeka.filter_items(regex, pathlist) -item_set_id = omeka.get_item_set() -is_public = True if omeka.get_environment() == "production" else False -#enables importing of overrides -sys.path.append(os.path.join(os.getcwd(), "scripts/overrides")) -post_items(pathlist) -#need to query the API again at this point so that records can be linked -omeka.reset() -link_items(pathlist) + ctx.record_error(OmekaAPIError(item_id, "link_records", err)) + return + + try: + ctx.client.update_resource(linked_item, "items") + except Exception as err: + # Log the full traceback at DEBUG level so that --log-level DEBUG + # reveals the exact API response; WARNING is shown by default. + logger.debug("Traceback for update_resource failure:", exc_info=True) + ctx.record_error(OmekaAPIError(item_id, "update_resource (link pass)", err)) + + +# --------------------------------------------------------------------------- +# Entrypoint +# --------------------------------------------------------------------------- + +def main(): + """ + Entrypoint: parse arguments, build context, run both passes, report. + + Execution order: + 1. Parse CLI arguments via _parse_args(). + 2. Configure the root logger (before any other work so all output is + captured at the right level). + 3. Build OmekaContext — loads config/private.yml, validates required keys, + initialises the authenticated API client. Exits with a descriptive + error message if the config is missing or malformed (OmekaConfigError). + 4. Discover JSON files under output//es/. + 5. Apply regex filter if -r was passed. + 6. Add scripts/python to sys.path so that collection-specific + omeka_overrides.py can be imported by field_definitions.get_fields(). + 7. Run pass 1 (post_items). + 8. Reset the API client between passes for a clean connection. + 9. Run pass 2 (link_items). + 10. Print run summary; exit 1 if any per-item errors were recorded, + 0 if all items succeeded. + """ + args = _parse_args() + + # Configure root logger first so that even OmekaContext initialisation + # errors are captured at the correct level. + configure_logging(args.log_level) + + # OmekaContext.from_args() raises OmekaConfigError (a subclass of + # OmekaError) if config/private.yml is missing, unparseable, or missing + # a required key. Let this propagate to the top level — configuration + # errors are fatal and should produce a clear traceback for the operator. + ctx = OmekaContext.from_args(args) + + # Resolve the ES output directory for the requested environment. + # ctx.resolve_path() returns an absolute Path relative to cwd (the + # collection root), so passing -e production reads from output/production/ + # rather than always defaulting to output/development/. + json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) + pathlist = list(Path(json_dir).glob("**/*.json")) + + if ctx.regex: + pathlist = filter_items(ctx.regex, pathlist) + + logger.info( + "Found %d JSON file(s) in %s (environment=%r)", + len(pathlist), + json_dir, + ctx.environment, + ) + + # Make the collection's scripts/python directory importable so that + # field_definitions.get_fields() can find omeka_overrides.py if present. + sys.path.append(os.path.join(os.getcwd(), "scripts/python")) + + # --- Pass 1: create / update items --- + logger.info("Starting pass 1: item posting") + post_items(ctx, pathlist) + + # Reset the API client between passes. ctx.reset_client() re-instantiates + # OmekaAPIClient with the same credentials, giving a fresh connection for + # the second round of requests. The property ID cache is preserved because + # term-to-ID mappings do not change between passes. + logger.info("Pass 1 complete. Resetting API client for pass 2.") + ctx.reset_client() + + # --- Pass 2: link related items --- + logger.info("Starting pass 2: record linking") + link_items(ctx, pathlist) + + # Print a consolidated summary of all per-item errors encountered during + # the run. Exits 0 if no errors; exits 1 if any item failed. The Ruby + # caller (bin/post_omeka) checks the exit code to determine whether the + # run completed cleanly. + ctx.report_errors() + sys.exit(1 if ctx._errors else 0) + +if __name__ == "__main__": + main() From 19251215081b0da70f65779b3f5ad305e6f59b38 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 16:43:40 -0500 Subject: [PATCH 048/222] wrap body in main(); fix hardcoded paths to use env; internal functions receive ctx; use logger --- lib/datura/python/html_and_media_ingest.py | 539 ++++++++++++++++----- 1 file changed, 423 insertions(+), 116 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 870646c78..31c257c4f 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -1,139 +1,446 @@ -import omeka -from pathlib import Path +""" +html_and_media_ingest.py + +Entrypoint script: attaches HTML and IIIF thumbnail media objects to items +that have already been posted to Omeka S by json_to_omeka.py. + +For each JSON record this script: +1. Looks up the Omeka item by its CDRH identifier. +2. Optionally deletes any existing media objects (unless --media-skip is set + and the item already has 2+ media objects). +3. Downloads the IIIF thumbnail from the configured iiif_server and uploads + it to Omeka as the primary media object (so Omeka designates it the + primary_media for the item). +4. Reads the pre-rendered HTML file from output//html/ and uploads it + as an HTML media object. + +Usage (from collection root directory): + python3 html_and_media_ingest.py -e development + python3 html_and_media_ingest.py -e production -r "some_pattern" + python3 html_and_media_ingest.py -e development -m # skip items that already have media + python3 html_and_media_ingest.py -e development --log-level DEBUG + +The script is invoked by bin/post_omeka_html in the Datura gem. The Ruby +wrapper passes -e, -r, and -m arguments from its own CLI. +""" + +import argparse import json -# not used, but needed for debugging +import logging import sys -import traceback +from pathlib import Path + import requests from requests.exceptions import HTTPError -from copy import deepcopy - -#look for the output folder: /output/development/* -json_dir = omeka.get_dir("output/development/es") -pathlist = list(Path(json_dir).glob('**/*.json')) -regex = omeka.get_regex() -if regex: - pathlist = omeka.filter_items(regex, pathlist) -html_dir = omeka.get_dir("output/development/html") -iiif_dir = omeka.get_dir("output/development/iiif") - -item_set_id = omeka.get_item_set() - -def delete_media_items(matching_item): - if len(matching_item["o:media"]) >= 1: - for media_item in matching_item["o:media"]: - try: - print("deleting media item " + str(media_item["o:id"])) - omeka.omeka_auth.delete_resource(media_item["o:id"], "media") - except HTTPError as err: - if err.response.status_code == 500: - continue - else: - print("error deleting media item: " + str(err)) - raise - -def ingest_thumbnail(json_item, matching_item): - ## IIIF THUMBNAIL INGEST - # note that thumbnail ingest should be done first so that thumbnails are designated primary_media - - collection_name = json_item["collection"] - cover_image = json_item.get("cover_image", None) + +import omeka +from omeka import add_media_to_item, filter_items +from omeka_context import ( + OmekaAPIError, + OmekaContext, + OmekaMediaError, + OmekaMultipleMatchesError, + OmekaItemNotFoundError, + configure_logging, +) + +# Module-level logger. Records from this module appear as +# "html_and_media_ingest" in log output. +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# CLI argument parsing +# --------------------------------------------------------------------------- + +def _parse_args(): + """ + Parse command-line arguments for the HTML/media ingest entrypoint. + + Returns an argparse.Namespace with: + * environment - "development" or "production" + * regex - optional file-filter pattern string, or None + * media_skip - bool; True skips items that already have 2+ media objects + * log_level - logging level string, default "INFO" + """ + parser = argparse.ArgumentParser( + description="Attach HTML and IIIF thumbnail media to existing Omeka S items." + ) + parser.add_argument( + "-e", "--environment", + required=True, + help="Target environment: 'development' or 'production'.", + ) + parser.add_argument( + "-r", "--regex", + default=None, + help=( + "Optional regex pattern to restrict processing to matching " + "file paths. Example: -r 'abc123' processes only files whose " + "path contains 'abc123'." + ), + ) + parser.add_argument( + "-m", "--media-skip", + action="store_true", + dest="media_skip", + help=( + "Skip re-ingesting media for items that already have 2 or more " + "media objects (thumbnail + HTML). Useful when re-running the " + "script after a partial failure to avoid re-uploading media that " + "was already successfully ingested." + ), + ) + parser.add_argument( + "--log-level", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + dest="log_level", + help="Set the logging verbosity (default: INFO).", + ) + return parser.parse_args() + + +# --------------------------------------------------------------------------- +# Media operations +# --------------------------------------------------------------------------- + +def delete_media_items(ctx, matching_item): + """ + Delete all media objects currently attached to an Omeka item. + + Called before re-uploading thumbnail and HTML so that the item does not + accumulate duplicate media objects across repeated script runs. + + HTTP 500 responses from the delete endpoint are treated as non-fatal — + the Omeka S API occasionally returns 500 for media items that have already + been removed in a prior step or that reference missing files on disk. + All other HTTP errors are recorded as OmekaMediaError and processing + continues with the next media object. + + Parameters: + * ctx - OmekaContext providing the authenticated API client + * matching_item - dict: the Omeka item JSON-LD object whose media to delete + """ + for media_item in matching_item.get("o:media", []): + media_id = media_item["o:id"] + try: + logger.info("Deleting media item %s", media_id) + ctx.client.delete_resource(media_id, "media") + except HTTPError as err: + if err.response.status_code == 500: + # 500 on DELETE is treated as "already gone" by convention. + # Log at DEBUG so it does not clutter normal output. + logger.debug( + "HTTP 500 deleting media %s (may already be absent); continuing", + media_id, + ) + else: + ctx.record_error( + OmekaMediaError( + "HTTP {} deleting media {}: {}".format( + err.response.status_code, media_id, err + ) + ) + ) + except Exception as err: + ctx.record_error( + OmekaMediaError("Unexpected error deleting media {}: {}".format(media_id, err)) + ) + + +def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): + """ + Download a IIIF thumbnail and upload it to Omeka as the item's primary media. + + Thumbnail ingest is performed before HTML ingest so that Omeka designates + the image as the item's primary_media (Omeka S uses the first media object + as the primary). + + If the source JSON record has no cover_image field, the function returns + immediately — not all items have thumbnails. + + If the thumbnail cannot be downloaded (network error, 4xx/5xx from the + IIIF server) or if the upload to Omeka fails, the failure is logged and + the function returns — the HTML ingest still proceeds. + + Parameters: + * ctx - OmekaContext (provides iiif_server, client, item_set_id) + * json_item - dict: one record from a Datura ES JSON file + * matching_item - dict: the Omeka item to attach the thumbnail to + * iiif_dir - pathlib.Path pointing to the local IIIF output directory + where the downloaded thumbnail is cached temporarily + """ + collection_name = json_item.get("collection", "") + cover_image = json_item.get("cover_image") + identifier = json_item.get("identifier", "unknown") + if not cover_image: + # No thumbnail configured for this item — nothing to do. + logger.debug("No cover_image for %r; skipping thumbnail ingest", identifier) return - # download thumbnail from iiif server - thumbnail_remote = f"{omeka.config['iiif_server']}/iiif/2/{collection_name}%2F{cover_image}.jpg/full/!200,200/0/default.jpg" - thumbnail_local = f"{iiif_dir}/{collection_name}%2F{cover_image}.jpg" + + # Construct the IIIF Image API URL for the thumbnail. + # The !200,200 size specifier requests a thumbnail that fits within a + # 200×200 bounding box while preserving aspect ratio. + thumbnail_remote = ( + "{}/iiif/2/{collection}%2F{image}.jpg/full/!200,200/0/default.jpg".format( + ctx.iiif_server, + collection=collection_name, + image=cover_image, + ) + ) + # Cache the thumbnail locally using the same URL-encoded filename so that + # re-runs can be inspected on disk if needed. + thumbnail_local = iiif_dir / "{}%2F{}.jpg".format(collection_name, cover_image) + + # --- Download --- try: - print(f"downloading thumbnail for {json_item['identifier']}") + logger.info("Downloading thumbnail for %r", identifier) response = requests.get(thumbnail_remote) response.raise_for_status() - with open (thumbnail_local, "wb") as thumb_file: + with open(thumbnail_local, "wb") as thumb_file: thumb_file.write(response.content) except Exception as err: - print(err) - print(f"error downloading thumbnail for {json_item['identifier']}, omitting") + logger.warning( + "Could not download thumbnail for %r: %s; skipping thumbnail ingest", + identifier, + err, + ) return - # attach thumbnail to api item - try: - with open(thumbnail_local, "rb") as thumb_file: - media_payload = { - "o:is_public": True, - "data": { - "upload": thumbnail_local, - "dcterms:title": omeka.prepare_property_value(json_item["title"], omeka.omeka_auth.get_property_id("dcterms:title")) - }, - "o:ingester": "upload" - } - print(f"posting thumbnail for {json_item['identifier']}") - try: - omeka.add_media_to_item(matching_item["o:id"], thumbnail_local, payload=media_payload) - except Exception as err: - print(err) - print(f"error adding image file for {json_item['identifier']}, omitting") - except FileNotFoundError: - print(f"file {thumbnail_local} not found, skipping thumbnail") -def ingest_html(json_item, matching_item): - #get desired path - file_path = f"{html_dir}/{json_item['identifier']}.html" - # get data from html + # --- Upload --- + # The title property ID is fetched via the cache so repeated calls for + # the same term do not make redundant API requests. try: - with open(file_path, "r") as file: - html_content = file.read() - if not html_content.strip(): - print(f"HTML file for {json_item['identifier']} is empty, skipping") - return media_payload = { - "o:is_public": True, + "o:is_public": ctx.is_public, "data": { - "html": html_content + "upload": str(thumbnail_local), + "dcterms:title": omeka.prepare_property_value( + json_item.get("title", ""), + ctx.get_property_id("dcterms:title"), + ), }, - "o:ingester": "html" + "o:ingester": "upload", } - print(f"posting html for {json_item['identifier']}") - try: - omeka.add_media_to_item(matching_item["o:id"], file_path, payload=media_payload) - except Exception as err: - print(err) - print(f"error adding html file for {json_item['identifier']}, omitting") - traceback.print_exc() + logger.info("Posting thumbnail for %r", identifier) + add_media_to_item(ctx, matching_item["o:id"], thumbnail_local, payload=media_payload) except FileNotFoundError: - print(f"file {file_path} not found, skipping item") + # The download step wrote the file, but something removed it between + # download and upload. Unlikely in practice but handled explicitly + # so the error message is clear. + logger.warning( + "Thumbnail file %s not found at upload time; skipping", + thumbnail_local, + ) + except Exception as err: + ctx.record_error( + OmekaMediaError( + "Error posting thumbnail for {!r}: {}".format(identifier, err) + ) + ) + + +def ingest_html(ctx, json_item, matching_item, html_dir): + """ + Read a pre-rendered HTML file and upload it to Omeka as an HTML media object. + + The HTML ingester reads content from payload["data"]["html"] rather than + from the uploaded file bytes; the file is opened only to read its content + into memory. The Omeka S "html" ingester stores the markup directly in + the database, making it searchable and renderable within Omeka. + + Skips silently if: + * The .html file does not exist at html_dir/.html. + * The file exists but is empty or contains only whitespace. + (An empty HTML file would create a blank media object in Omeka; this + guard prevents that. The root cause — an XSLT transform producing empty + output — should be investigated in the Datura XSLT/transform layer.) + + Parameters: + * ctx - OmekaContext + * json_item - dict: one record from a Datura ES JSON file + * matching_item - dict: the Omeka item to attach the HTML to + * html_dir - pathlib.Path pointing to the HTML output directory + """ + identifier = json_item.get("identifier", "unknown") + file_path = html_dir / "{}.html".format(identifier) + + try: + with open(file_path, "r") as file: + html_content = file.read() + except FileNotFoundError: + # A missing HTML file is common for items that have no text + # representation (e.g. pure image records). Log at INFO so operators + # can see which items were skipped without it being alarming. + logger.info("HTML file %s not found; skipping", file_path) + return + + # Guard against empty or whitespace-only files. An empty POST would + # create a blank HTML media object in Omeka, which is both incorrect and + # misleading when viewing the item in the admin UI. + if not html_content.strip(): + logger.warning( + "HTML file for %r is empty; skipping. " + "Check whether the XSLT transform produced output for this item.", + identifier, + ) + return + + media_payload = { + "o:is_public": ctx.is_public, + "data": { + "html": html_content, + }, + "o:ingester": "html", + } + + try: + logger.info("Posting HTML for %r", identifier) + add_media_to_item(ctx, matching_item["o:id"], file_path, payload=media_payload) + except Exception as err: + ctx.record_error( + OmekaMediaError( + "Error posting HTML for {!r}: {}".format(identifier, err) + ) + ) + + +# --------------------------------------------------------------------------- +# Main processing loop +# --------------------------------------------------------------------------- + +def process_items(ctx, pathlist, html_dir, iiif_dir): + """ + For each JSON record, look up the Omeka item and ingest its media. + + Logic: + * Skip items with no identifier (cannot look up in Omeka). + * Skip items with 0 or >1 Omeka matches (not posted / data integrity issue). + * If --media-skip is set and the item already has 2+ media objects + (thumbnail + HTML), skip re-ingestion to avoid unnecessary deletions. + * Otherwise: delete existing media, ingest thumbnail, ingest HTML. + + Parameters: + * ctx - OmekaContext + * pathlist - list of pathlib.Path objects for ES JSON files + * html_dir - pathlib.Path to output//html/ + * iiif_dir - pathlib.Path to output//iiif/ + """ + for path in pathlist: + filename = str(path) + with open(filename) as jsonfile: + json_items = json.load(jsonfile) -#iterate through each file -for path in pathlist: - filename = str(path) - with open(filename) as jsonfile: - json_items = json.load(jsonfile) for json_item in json_items: - if not json_item["identifier"]: + identifier = json_item.get("identifier") + if not identifier: + logger.debug("Skipping item without identifier in %s", filename) + continue + + # --- Look up the item in Omeka --- + try: + matching_items = ctx.client.filter_items_by_property( + filter_property="dcterms:identifier", + filter_value=identifier, + item_set_id=ctx.item_set_id, + ) + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "filter_items (media)", err)) + continue + + if not matching_items: + logger.warning( + "Unexpected empty response from filter_items for %r; skipping", + identifier, + ) + continue + + total = matching_items.get("total_results", 0) + if total == 0: + logger.warning("No Omeka item found for %r; skipping media ingest", identifier) + continue + if total > 1: + logger.warning( + "Multiple Omeka items (%d) found for %r; check admin site and skip", + total, + identifier, + ) continue - matching_items = omeka.omeka_auth.filter_items_by_property(filter_property = "dcterms:identifier", filter_value = json_item["identifier"], item_set_id=item_set_id) - if matching_items: - if matching_items["total_results"] == 1: - matching_item = matching_items["results"][0] - media_count = len(matching_item["o:media"]) - elif matching_items["total_results"] > 1: - print("multiple items found for " + json_item["identifier"] + ", check admin site") - continue - else: - print("no matching items for " + json_item["identifier"] + ", skipping") - continue - #check for existing media items, to avoid duplicates - #skip with -m flag - if not(omeka.args.media_skip and media_count >=2): - delete_media_items(matching_item) - - ## IIIF THUMBNAIL INGEST - #if -m flag, ingest only if not already present - # note that thumbnail ingest should be done first so that thumbnails are designated primary_media - ingest_thumbnail(json_item, matching_item) - - ## HTML INGEST - #if -m flag, ingest only if not already present - ingest_html(json_item, matching_item) - - ##TODO add other media ingest as needed - else: - print("skipping media for " + json_item["identifier"] + ", already ingested.") + matching_item = matching_items["results"][0] + media_count = len(matching_item.get("o:media", [])) + + # --media-skip: if the item already has 2+ media objects + # (thumbnail + HTML), assume it was already fully ingested and + # skip it to avoid redundant deletion and re-upload. + if ctx.media_skip and media_count >= 2: + logger.info( + "Skipping media for %r: already has %d media object(s)", + identifier, + media_count, + ) + continue + + # --- Media pipeline --- + # Delete first, then re-upload. Order matters: thumbnail must be + # uploaded before HTML so that Omeka designates the image as + # primary_media. + delete_media_items(ctx, matching_item) + ingest_thumbnail(ctx, json_item, matching_item, iiif_dir) + ingest_html(ctx, json_item, matching_item, html_dir) + + +# --------------------------------------------------------------------------- +# Entrypoint +# --------------------------------------------------------------------------- + +def main(): + """ + Entrypoint: parse arguments, build context, run media ingest, report. + + Execution order: + 1. Parse CLI arguments. + 2. Configure root logger. + 3. Build OmekaContext (loads config, validates keys, creates API client). + 4. Resolve output directories for the requested environment. + 5. Discover JSON files; apply regex filter if -r was passed. + 6. Run media ingest for all items. + 7. Report errors; exit 1 if any failures, 0 if clean. + """ + args = _parse_args() + configure_logging(args.log_level) + + # OmekaConfigError propagates here as a fatal error — missing or broken + # config means no API access is possible. + ctx = OmekaContext.from_args(args) + + # Resolve all three environment-specific directories using the requested + # environment so that -e production reads from output/production/ rather + # than always defaulting to output/development/. + json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) + html_dir = ctx.resolve_path("output/{}/html".format(ctx.environment)) + iiif_dir = ctx.resolve_path("output/{}/iiif".format(ctx.environment)) + + pathlist = list(Path(json_dir).glob("**/*.json")) + + if ctx.regex: + pathlist = filter_items(ctx.regex, pathlist) + + logger.info( + "Found %d JSON file(s) in %s (environment=%r, media_skip=%s)", + len(pathlist), + json_dir, + ctx.environment, + ctx.media_skip, + ) + + process_items(ctx, pathlist, html_dir, iiif_dir) + + ctx.report_errors() + sys.exit(1 if ctx._errors else 0) + + +if __name__ == "__main__": + main() From 7b7b06a9fb2730869751e6278a989aebcf281a9a Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 8 May 2026 17:01:07 -0500 Subject: [PATCH 049/222] add date filter to omeka transformation and posting --- lib/datura/python/html_and_media_ingest.py | 14 +++++- lib/datura/python/json_to_omeka.py | 14 +++++- lib/datura/python/omeka.py | 53 ++++++++++++++++++++++ lib/datura/python/omeka_context.py | 41 ++++++++++++++++- 4 files changed, 119 insertions(+), 3 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 31c257c4f..deff3ff05 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -34,7 +34,7 @@ from requests.exceptions import HTTPError import omeka -from omeka import add_media_to_item, filter_items +from omeka import add_media_to_item, filter_items, filter_items_by_date from omeka_context import ( OmekaAPIError, OmekaContext, @@ -80,6 +80,16 @@ def _parse_args(): "path contains 'abc123'." ), ) + parser.add_argument( + "-u", "--update", + default=None, + dest="update_time", + help=( + "Only process items whose source file was modified at or after " + "this date/time. Accepts 'today', a date (2015-01-01), or " + "date-time (2015-01-01T18:24)." + ), + ) parser.add_argument( "-m", "--media-skip", action="store_true", @@ -427,6 +437,8 @@ def main(): if ctx.regex: pathlist = filter_items(ctx.regex, pathlist) + if ctx.update_time: + pathlist = filter_items_by_date(ctx.update_time, pathlist) logger.info( "Found %d JSON file(s) in %s (environment=%r, media_skip=%s)", diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 8fb366806..6e9816a63 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -46,7 +46,7 @@ OmekaMultipleMatchesError, configure_logging, ) -from omeka import filter_items, prepare_item_payload_using_template +from omeka import filter_items, filter_items_by_date, prepare_item_payload_using_template # Module-level logger. Records from this module appear as "json_to_omeka" # in log output so they can be filtered independently from other modules. @@ -88,6 +88,16 @@ def _parse_args(): "path contains 'abc123'." ), ) + parser.add_argument( + "-u", "--update", + default=None, + dest="update_time", + help=( + "Only process items whose source file was modified at or after " + "this date/time. Accepts 'today', a date (2015-01-01), or " + "date-time (2015-01-01T18:24)." + ), + ) parser.add_argument( "--log-level", default="INFO", @@ -411,6 +421,8 @@ def main(): if ctx.regex: pathlist = filter_items(ctx.regex, pathlist) + if ctx.update_time: + pathlist = filter_items_by_date(ctx.update_time, pathlist) logger.info( "Found %d JSON file(s) in %s (environment=%r)", diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 5c2a2a40e..2fbbb4b71 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -5,8 +5,10 @@ """ +from datetime import datetime from pathlib import Path import json +import os import re @@ -252,3 +254,54 @@ def filter_items(regex, pathlist): """ reg = re.compile(regex) return [p for p in pathlist if reg.search(str(p))] + +def filter_items(regex, pathlist): + """ + Filter a list of file paths to those matching a regex pattern. + + Used by both entrypoint scripts to restrict processing to a subset of + files when the -r / --regex flag is passed on the command line. + + Parameters: + * regex - regex pattern string, compiled with re.compile() + * pathlist - iterable of pathlib.Path or string paths to filter + + Returns a list containing only the paths whose string representation + matches the pattern. + """ + reg = re.compile(regex) + return [p for p in pathlist if reg.search(str(p))] + + +def filter_items_by_date(update_time, pathlist): + """ + Filter a list of output JSON file paths to those whose corresponding source + file has a modification time at or after update_time. + + Source files are expected at source//. relative to + the collection root (cwd), matching the Datura convention where the output + JSON stem equals the source filename stem (e.g. source/tei/abc123.xml -> + output/development/es/abc123.json). Items with no locatable source file are + included unconditionally so they are not silently dropped. + + Parameters: + * update_time - datetime object; only items with source mtime >= this are kept + * pathlist - iterable of pathlib.Path or string paths to filter + """ + source_base = Path.cwd() / "source" + result = [] + for p in pathlist: + identifier = Path(str(p)).stem + source_files = list(source_base.glob("*/{}.*".format(identifier))) + if not source_files: + logger.debug( + "No source file found for %r; including without date filter", identifier + ) + result.append(p) + continue + source_mtime = max( + datetime.fromtimestamp(os.path.getmtime(str(sf))) for sf in source_files + ) + if source_mtime >= update_time: + result.append(p) + return result diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 661001b81..638bd9d1c 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -7,6 +7,7 @@ import logging import sys +from datetime import date, datetime from pathlib import Path from typing import Dict, List, Optional @@ -125,6 +126,37 @@ class OmekaMediaError(OmekaError): """ +# --------------------------------------------------------------------------- +# Date parsing +# --------------------------------------------------------------------------- + +def parse_update_time(s): + """ + Parse a -u / --update date string into a datetime object. + + Accepts the same formats as the Ruby Datura -u flag: + * "today" - midnight of the current local date + * "2015-01-01" - date only + * "2015-01-01T18:24" - date and time + + Raises OmekaConfigError with a descriptive message if the string does not + match any expected format. + """ + if s == "today": + d = date.today() + return datetime(d.year, d.month, d.day) + for fmt in ("%Y-%m-%dT%H:%M", "%Y-%m-%d"): + try: + return datetime.strptime(s, fmt) + except ValueError: + continue + raise OmekaConfigError( + "Invalid --update value {!r}. " + "Expected 'today', a date (2015-01-01), or date-time (2015-01-01T18:24)." + .format(s) + ) + + # --------------------------------------------------------------------------- # Context # --------------------------------------------------------------------------- @@ -159,6 +191,7 @@ def from_args(cls, args): Expected attributes: .environment str "development" or "production" .regex str optional file-filter pattern, or None + .update_time str optional date/time string for -u filter, or None .media_skip bool skip re-ingesting existing media (html_and_media_ingest only; absent on json_to_omeka args, defaults to False) @@ -187,6 +220,7 @@ def from_args(cls, args): ) env_config = {} + raw_update = getattr(args, "update_time", None) return cls( config=default_config, env_config=env_config, @@ -195,6 +229,7 @@ def from_args(cls, args): # every flag (e.g. json_to_omeka.py has no --media-skip). regex=getattr(args, "regex", None), media_skip=getattr(args, "media_skip", False), + update_time=parse_update_time(raw_update) if raw_update else None, ) @staticmethod @@ -236,7 +271,7 @@ def _load_config(path, env): return contents[env] - def __init__(self, config, env_config, environment, regex, media_skip): + def __init__(self, config, env_config, environment, regex, media_skip, update_time=None)): """ Initialise the context. Prefer OmekaContext.from_args() over calling this constructor directly except in tests. @@ -252,6 +287,9 @@ def __init__(self, config, env_config, environment, regex, media_skip): None means process all files in the output directory * media_skip - if True, items that already have 2+ media objects (thumbnail + HTML) are skipped during media ingest + * update_time - optional datetime; if set, only items whose source file + mtime >= this value are processed (mirrors the -u flag + from the main Datura post command) """ # ---- Validate required config keys -------------------------------- # Validate up front so that failures are immediate and descriptive. @@ -276,6 +314,7 @@ def __init__(self, config, env_config, environment, regex, media_skip): self.environment = environment self.regex = regex self.media_skip = media_skip + self.update_time = update_time # ---- Config values ------------------------------------------------ self.template_number = config["resource_template"] From fb3461457007f9f9e80b89c7b63508d913f66cd8 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 09:33:38 -0500 Subject: [PATCH 050/222] use safe_load_file instead of safe_load with File.read --- lib/datura/elasticsearch/index.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index fe5e1b890..8010a6385 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -25,7 +25,7 @@ def initialize(options = nil, schema_mapping: false) @mapping_url = File.join(@index_url, "_mapping?pretty=true") # yaml settings (if exist) and mappings - @requested_schema = YAML.safe_load(File.read(@options["es_schema"]), permitted_classes: [Symbol]) + @requested_schema = YAML.safe_load_file(@options["es_schema"], permitted_classes: [Symbol]) @auth_header = Datura::Helpers.construct_auth_header(@options) # if requested, grab the mapping currently associated with this index # otherwise wait until after the requested schema is loaded From 95647a17c19f410de45552cab3bed6b9fb13d4b5 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 10:05:56 -0500 Subject: [PATCH 051/222] use safe_load_file instead of safe_load with File.read --- lib/datura/options.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/options.rb b/lib/datura/options.rb index 4987069b2..c37db9916 100644 --- a/lib/datura/options.rb +++ b/lib/datura/options.rb @@ -64,7 +64,7 @@ def read_all_configs(general, collection) def read_config(path) if File.file?(path) begin - return YAML.safe_load(File.read(path), permitted_classes: [Symbol]) + return YAML.safe_load_file(path, permitted_classes: [Symbol]) rescue Exception => e puts "There was an error reading config file #{path}: #{e.message}" end From 6eaa5843bfc19e4abd00b3408f3ea20fa13ba89e Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 10:24:07 -0500 Subject: [PATCH 052/222] align HTTP request variables, remove unnecessary setting of http.verify_mode --- lib/datura/helpers.rb | 1 - lib/datura/solr_poster.rb | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 8484e6b0f..c0ed329f9 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -78,7 +78,6 @@ def self.get_url(url) http = Net::HTTP.new(uri.host, uri.port) if uri.scheme == "https" http.use_ssl = true - http.verify_mode = OpenSSL::SSL::VERIFY_PEER end http.request(Net::HTTP::Get.new(uri.request_uri)) end diff --git a/lib/datura/solr_poster.rb b/lib/datura/solr_poster.rb index 84a21861a..7a39aa417 100644 --- a/lib/datura/solr_poster.rb +++ b/lib/datura/solr_poster.rb @@ -53,12 +53,11 @@ def commit_solr end def post(content, type) - url = URI.parse(@url) - http = Net::HTTP.new(url.host, url.port) + uri = URI.parse(@url) + http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = @url.start_with?("https") - http.verify_mode = OpenSSL::SSL::VERIFY_PEER http.open_timeout = 10 - request = Net::HTTP::Post.new(url.request_uri) + request = Net::HTTP::Post.new(uri.request_uri) request.body = content request["Content-Type"] = type http.request(request) From 9237fe3806f259cbf2c8ab551518d5b903bcaf5c Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 11:40:39 -0500 Subject: [PATCH 053/222] add warning message if regex match does not exist --- lib/datura/data_manager.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 9ce1022d6..ceca339d5 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -249,6 +249,11 @@ def prepare_files allowed = allowed_files(files) # filter by regex regexed = Datura::Helpers.regex_files(allowed, @options["regex"]) + if @options["regex"] && regexed.empty? + msg = "No files matched regex: #{@options['regex']}" + puts msg.yellow + @log.warn(msg) + end # filter by date filtered = regexed.select { |f| Datura::Helpers.should_update?(f, @options["update_time"]) } From cbb0e2decbe6a0f365f2b408ad0bab020f72002a Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 12:09:21 -0500 Subject: [PATCH 054/222] remove commented breakpoints and sys.exit(1)s, so script can proceed after item errors --- lib/datura/python/api_fields.py | 3 --- lib/datura/python/json_to_omeka.py | 10 ---------- 2 files changed, 13 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index b5b3669cd..2cc9d339a 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -72,9 +72,7 @@ def build_item_dict(json, existing_item): update_item_value(built_item, "dh:itemText", fields.itemText(json)) return built_item except ValueError as e: - #breakpoint() print(f"Error: {e}", file=sys.stderr) - sys.exit(1) #TODO change item linking for JSON and new API @@ -245,7 +243,6 @@ def get_omeka_ids(lookup_values, filter_property, item_set_id = None): if match["total_results"] >= 1: if match["total_results"] > 1: print(f"warning: multiple matches for {lookup_value}, taking first match") - #breakpoint() omeka_id = match['results'][0]["o:id"] omeka_ids.append(omeka_id) else: diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 03528a7fe..cedebd7b7 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -21,13 +21,10 @@ def post_items(pathlist): for json_item in json_items: try: if not json_item["identifier"]: - #breakpoint() print("skipping item without identifier") continue except TypeError as e: - #breakpoint() print(f"Error: {e}", file=sys.stderr) - sys.exit(1) matching_items = omeka.omeka_auth.filter_items_by_property(filter_property = "dcterms:identifier", filter_value = json_item["identifier"], item_set_id=item_set_id) if matching_items: @@ -70,7 +67,6 @@ def link_item(json_item, matching_items): print(str(err)) traceback.print_exc print(f"Error updating item {item_id}") - #breakpoint() pass def add_new_item(json_item, template_number): @@ -80,16 +76,12 @@ def add_new_item(json_item, template_number): print(f"creating item {new_item['dcterms:identifier'][0]['@value']}") except KeyError as err: print(err) - #breakpoint() - sys.exit(1) payload = omeka.prepare_item_payload_using_template(new_item, template_number) # add item set try: omeka.omeka_auth.add_item(payload, template_id=template_number, item_set_id=item_set_id, is_public=is_public) except Exception as err: print(err) - #breakpoint() - sys.exit(1) else: print(f"error preparing item {json_item['identifier']}") @@ -102,8 +94,6 @@ def update_existing_item(json_item, matching_items): omeka.omeka_auth.update_resource(updated_item, "items") except Exception as err: print(err) - #breakpoint() - sys.exit(1) #look for the output folder: /output/development/es and get all items json_dir = omeka.get_dir("output/development/es") From 3988681cb2caa833b0cbb87be77417409bfde82d Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 15:09:39 -0500 Subject: [PATCH 055/222] add row_filter to build_html_from_csv --- lib/datura/file_types/file_csv.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index 9d2954ab4..4783a26ef 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -7,9 +7,13 @@ def initialize(file_location, options) @csv = read_csv(file_location, options["csv_encoding"]) end - def build_html_from_csv + # row_filter is an optional regexp; when present, only rows whose identifier + # matches the pattern are converted to HTML. Pass nil to process all rows. + def build_html_from_csv(row_filter = nil) @csv.each_with_index do |row, index| next if row.header_row? + # Skip rows that don't match the identifier filter (if one is active) + next if row_filter && !row_matches_filter?(row, row_filter) # Note: if overriding this function, it's recommended to use # a more specific identifier for each row of the CSV # but since this is a generic version, simply using the current iteration number From 8262c3824e3b39395576e454eb9511f429c6aacb Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 15:11:11 -0500 Subject: [PATCH 056/222] add row_filter notification to transform_html --- lib/datura/file_types/file_csv.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index 4783a26ef..0024a4e86 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -103,6 +103,12 @@ def transform_iiif def transform_html puts "transforming #{self.filename} to HTML subdocuments" + # Build and apply the identifier filter, if --csv-rows was passed + row_filter = build_csv_row_filter + if row_filter + puts "csv_rows filter active: only processing rows matching /#{@options["csv_rows"]}/".cyan + end + build_html_from_csv(row_filter) build_html_from_csv # transform_html method is expected to send back a hash # but already wrote to filesystem so just sending back empty From 7fa412baa7b452ecb2412f0a8a81d48f10ab571c Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 15:13:36 -0500 Subject: [PATCH 057/222] add row_filter to transform_solr --- lib/datura/file_types/file_csv.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index 0024a4e86..e507fec75 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -120,14 +120,20 @@ def transform_html # it will have to do! (transmississippi only collection so far) def transform_solr puts "transforming #{self.filename}" + # Build and apply the identifier filter, if --csv-rows was passed + row_filter = build_csv_row_filter + if row_filter + puts "csv_rows filter active: only processing rows matching /#{@options["csv_rows"]}/".cyan + end solr_doc = Nokogiri::XML("") @csv.each do |row| - if !row.header_row? - doc = Nokogiri::XML::Node.new("doc", solr_doc) - # row_to_solr should return an XML::Node object with children - doc = row_to_solr(doc, @csv.headers, row) - solr_doc.at_css("add").add_child(doc) - end + next if row.header_row? + # Skip rows that don't match the identifier filter (if one is active) + next if row_filter && !row_matches_filter?(row, row_filter) + doc = Nokogiri::XML::Node.new("doc", solr_doc) + # row_to_solr should return an XML::Node object with children + doc = row_to_solr(doc, @csv.headers, row) + solr_doc.at_css("add").add_child(doc) end # Uncomment to debug # puts solr_doc.root.to_xml From 248ac9b41619a3a17d0123839632d40614dfbd66 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 15:19:06 -0500 Subject: [PATCH 058/222] add csv filtering to omeka transformation --- bin/post_omeka | 9 +++++++++ bin/post_omeka_html | 9 +++++++++ lib/datura/python/html_and_media_ingest.py | 4 ++++ lib/datura/python/json_to_omeka.py | 9 +++++++++ lib/datura/python/omeka.py | 21 +++++++++++++++++++-- 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index bba61370a..09ec7e298 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -29,6 +29,12 @@ optparse = OptionParser.new do |opts| end end + opts.on('-c', '--csv-rows [input]', 'only process CSV rows matching this identifier regex') do |input| + if input && input.length > 0 + options["csv_rows"] = input + end + end + opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| #does nothing, this is a placeholder to catch all arguments not needed now and pass them along to the main script end @@ -54,6 +60,9 @@ if File.exist?("#{python_script_path}") if options["regex"] command.append("-r", Shellwords.escape(options["regex"])) end + if options["csv_rows"] + command.append("-c", Shellwords.escape(options["csv_rows"])) + end system(*command) else puts("Omeka script not found at #{python_script_path}".red) diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 27f3a3522..e1e1287fa 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -29,6 +29,12 @@ optparse = OptionParser.new do |opts| end end + opts.on('-c', '--csv-rows [input]', 'only process CSV rows matching this identifier regex') do |input| + if input && input.length > 0 + options["csv_rows"] = input + end + end + opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| #does nothing, this is a placeholder to catch all arguments not needed now and pass them along to the main script end @@ -57,6 +63,9 @@ if File.exist?("#{python_script_path}") if options["regex"] command.append("-r", Shellwords.escape(options["regex"])) end + if options["csv_rows"] + command.append("-c", Shellwords.escape(options["csv_rows"])) + end if options["media_skip"] command.append("-m") end diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 6dbbce353..a23e591ec 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -102,6 +102,10 @@ def ingest_html(json_item, matching_item): filename = str(path) with open(filename) as jsonfile: json_items = json.load(jsonfile) + # Apply --csv-rows identifier filter if provided. + csv_rows = omeka.get_csv_rows() + if csv_rows: + json_items = omeka.filter_items_by_identifier(csv_rows, json_items) for json_item in json_items: if not json_item["identifier"]: continue diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 5d2f63d8d..0be2dae24 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -16,6 +16,10 @@ def post_items(pathlist): filename = str(path) with open(filename) as jsonfile: json_items = json.load(jsonfile) + # Apply --csv-rows identifier filter if provided. + csv_rows = omeka.get_csv_rows() + if csv_rows: + json_items = omeka.filter_items_by_identifier(csv_rows, json_items) # TODO change template_number to actual number, account for other schemas is necessary template_number = omeka.template_number for json_item in json_items: @@ -44,6 +48,11 @@ def link_items(pathlist): filename = str(path) with open(filename) as jsonfile: json_items = json.load(jsonfile) + # Apply the same --csv-rows filter used in post_items so that only + # the targeted rows are linked + csv_rows = omeka.get_csv_rows() + if csv_rows: + json_items = omeka.filter_items_by_identifier(csv_rows, json_items) for json_item in json_items: if not json_item["identifier"]: print("skipping item without identifier") diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 5b079d04f..12969e4eb 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -54,8 +54,23 @@ def get_environment(): return environment def get_regex(): - regex = args.regex - return regex + # Return the --regex string, or None if not set. + return args.regex + +def get_csv_rows(): + # Return the --csv-rows regex string, or None if not set. + return args.csv_rows + +def filter_items_by_identifier(csv_rows_regex, json_items): + '''Filter a list of JSON item dicts to only those whose identifier matches csv_rows_regex. + + Used by the Omeka posting scripts to apply the same --csv-rows filter that + the Ruby layer applies during ES/HTML/Solr generation. Because CSV input + produces a single multi-item JSON array (unlike TEI/VRA), filtering must + happen here as well. + ''' + reg = re.compile(csv_rows_regex) + return [item for item in json_items if reg.search(item.get("identifier", ""))] def add_media_to_item(item_id, media_file, payload={}, template_id=None, class_id=None): # copied from the module to modify with different ingester @@ -218,6 +233,8 @@ def filter_items(regex, pathlist): parser.add_argument('-m', '--media-skip', action='store_true', help='Only ingest media not already ingested') parser.add_argument('-r', '--regex', required=False, help = "Filter files with regex") +parser.add_argument('-c', '--csv-rows', required=False, + help='Only process CSV items whose identifier matches this regex') args = parser.parse_args() template_number = config["resource_template"] omeka_data_base = config["omeka_data_base"] \ No newline at end of file From 4bc4f54bf22b1885fc9747e713d153409bc46a18 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 15:54:58 -0500 Subject: [PATCH 059/222] fix csv filtering in html transformation --- lib/datura/file_types/file_csv.rb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index e507fec75..e0036adc2 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -7,9 +7,15 @@ def initialize(file_location, options) @csv = read_csv(file_location, options["csv_encoding"]) end - # row_filter is an optional regexp; when present, only rows whose identifier - # matches the pattern are converted to HTML. Pass nil to process all rows. - def build_html_from_csv(row_filter = nil) + # Builds one HTML file per CSV row. Respects the --csv-rows filter option: + # if @options["csv_rows"] is set, only rows with identifier matching that + # regex are written. Note: if overriding this method in a collection, call + # build_csv_row_filter / row_matches_filter? to preserve filter behavior. + def build_html_from_csv + row_filter = build_csv_row_filter + if row_filter + puts "csv_rows filter active: only processing rows matching /#{@options["csv_rows"]}/".cyan + end @csv.each_with_index do |row, index| next if row.header_row? # Skip rows that don't match the identifier filter (if one is active) @@ -103,12 +109,7 @@ def transform_iiif def transform_html puts "transforming #{self.filename} to HTML subdocuments" - # Build and apply the identifier filter, if --csv-rows was passed - row_filter = build_csv_row_filter - if row_filter - puts "csv_rows filter active: only processing rows matching /#{@options["csv_rows"]}/".cyan - end - build_html_from_csv(row_filter) + # build_html_from_csv handles the --csv-rows filter internally build_html_from_csv # transform_html method is expected to send back a hash # but already wrote to filesystem so just sending back empty From d775c52ea4c804eee2b8bb354e3ef210098f68ef Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 13 May 2026 16:10:12 -0500 Subject: [PATCH 060/222] reverse earlier interpolation edit, which does not play well with updates to nokogiri --- lib/datura/file_types/file_csv.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index e0036adc2..b271ac704 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -65,11 +65,7 @@ def row_to_es(headers, row) # operates with no logic on the fields def row_to_solr(doc, headers, row) headers.each do |column| - next unless row[column] - field = Nokogiri::XML::Node.new("field", doc) - field["name"] = column - field.content = row[column] - doc.add_child(field) + doc.add_child("#{row[column]}") if row[column] end doc end From 1e1d0abe3afe9998b6280ebb12f839064a4ca9b3 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 14 May 2026 09:49:48 -0500 Subject: [PATCH 061/222] add csv row filter to posting docs --- docs/3_manage/post.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index 6ba7b4a2f..1e67c45f9 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -28,6 +28,14 @@ The above does the following: Displays usage and list of options +``` +-c, --csv-rows [input] +``` + +Transforms / posts only csv lines whose identifier (id/identifier column) matches a specific regular expression. + +Examples: `post -c foo_001 (exact), foo_ (prefix), 'foo_00[1-3]' (range), 'foo_\d' (use digit character)` + ``` -e, --environment [input] ``` From e5d47500e70f3d07c500aebe4491dce7e6980469 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 14 May 2026 11:07:47 -0500 Subject: [PATCH 062/222] add base-output-uri to cmd in cases of xsl:result-document outputs; update path in xslt --- lib/datura/file_type.rb | 3 ++- lib/datura/python/xslt_transform.py | 16 ++++++++++++++-- .../lib/personography_encyclopedia.xsl | 5 ++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index ad372131b..e05fa4deb 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -200,9 +200,10 @@ def exec_xsl(input, xsl, ext, outpath=nil, params=nil) cmd += ["--param", "#{k}=#{v}"] end end - # append output path + # append output path and base output URI for xsl:result-document secondary outputs if configured if outpath cmd += ["--output", "#{outpath}/#{filename(false)}.#{ext}"] + cmd += ["--base-output-uri", outpath] end puts "using command #{cmd.inspect}" if @options["verbose"] # run the command diff --git a/lib/datura/python/xslt_transform.py b/lib/datura/python/xslt_transform.py index 1c429e50d..24fc2a2ff 100644 --- a/lib/datura/python/xslt_transform.py +++ b/lib/datura/python/xslt_transform.py @@ -13,11 +13,13 @@ def parse_args(): parser.add_argument("--output", required=False, help="Path to write output file") parser.add_argument("--param", action="append", default=[], metavar="KEY=VALUE", help="XSL parameter (repeatable)") + parser.add_argument("--base-output-uri", required=False, dest="base_output_uri", + help="Base URI for xsl:result-document secondary outputs (file:// URI or directory path)") # parse sys.argv, validate args, return namespace object with attributes return parser.parse_args() -def run_transform(input_path, xsl_path, params, output_path=None): +def run_transform(input_path, xsl_path, params, output_path=None, base_output_uri=None): # import here rather than at top so error is raised at call time with clear traceback if saxonche isn't installed import saxonche # create Saxon processor using Home Edition tier @@ -32,10 +34,19 @@ def run_transform(input_path, xsl_path, params, output_path=None): xslt_proc.set_parameter(key, proc.make_string_value(value)) # parse and compile xsl file into executable executable = xslt_proc.compile_stylesheet(stylesheet_file=xsl_path) + # set base output URI for xsl:result-document secondary outputs if provided; + # convert plain directory path to file:// URI if needed + if base_output_uri: + if not base_output_uri.startswith("file://"): + base_output_uri = Path(base_output_uri).resolve().as_uri() + "/" + executable.set_base_output_uri(base_output_uri) # run transformation and return output as string result = executable.transform_to_string(source_file=input_path) - # return error or write output to disk + # when a base output URI is set, None primary output is expected — all output + # went to xsl:result-document secondary files; only error when neither was produced if result is None: + if base_output_uri: + return raise RuntimeError("Transformation produced no output") if output_path: Path(output_path).write_text(result, encoding="utf-8") @@ -51,6 +62,7 @@ def main(): xsl_path=args.xsl, params=args.param, output_path=args.output, + base_output_uri=args.base_output_uri, ) except Exception as e: print(f"XSLT transformation error: {e}", file=sys.stderr) diff --git a/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl b/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl index d15066563..c833f082e 100644 --- a/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl +++ b/lib/xslt/tei_to_html/lib/personography_encyclopedia.xsl @@ -39,9 +39,8 @@ - - - + + From a054f1f71afafff128b35ccb67cdc2c2888bf85e Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 14 May 2026 11:10:05 -0500 Subject: [PATCH 063/222] remove defunct link --- docs/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 75618aa9d..3d7b0c2e7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,7 +35,6 @@ The files are parsed and formatted into documents appropriate for Solr, IIIF, El - Remove / destroy index - Developers - [Installation](4_developers/installation.md) - - [Saxon setup](4_developers/saxon.md) - [Ruby / Gems](4_developers/ruby_gems.md) - Class organization - [Tests](4_developers/test.md) From b40ef43c51dce27b52db1dbabf075d5d4fe3744c Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 14 May 2026 11:22:09 -0500 Subject: [PATCH 064/222] switch to more recognizable example text --- docs/3_manage/post.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index 530eeb492..a26f0d0f5 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -36,7 +36,7 @@ Displays usage and list of options Transforms / posts only csv lines whose identifier (id/identifier column) matches a specific regular expression. -Examples: `post -c foo_001 (exact), foo_ (prefix), 'foo_00[1-3]' (range), 'foo_\d' (use digit character)` +Examples: `post -c cat_001 (exact), cat_ (prefix), 'cat_00[1-3]' (range), 'cat_\d' (use digit character)` ```bash -e, --environment [input] From 66efb6032226eff36c358e2eb7ece78abb84a88b Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 14 May 2026 13:33:32 -0500 Subject: [PATCH 065/222] standardize and simplify ssl flag-setting --- lib/datura/helpers.rb | 4 +--- lib/datura/solr_poster.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index c0ed329f9..671c1511c 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -76,9 +76,7 @@ def self.get_input(original_input, msg) def self.get_url(url) uri = URI.parse(url) http = Net::HTTP.new(uri.host, uri.port) - if uri.scheme == "https" - http.use_ssl = true - end + http.use_ssl = true if uri.scheme == "https" http.request(Net::HTTP::Get.new(uri.request_uri)) end diff --git a/lib/datura/solr_poster.rb b/lib/datura/solr_poster.rb index 7a39aa417..7049330ec 100644 --- a/lib/datura/solr_poster.rb +++ b/lib/datura/solr_poster.rb @@ -55,7 +55,7 @@ def commit_solr def post(content, type) uri = URI.parse(@url) http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = @url.start_with?("https") + http.use_ssl = true if uri.scheme == "https" http.open_timeout = 10 request = Net::HTTP::Post.new(uri.request_uri) request.body = content From 2299655a3ea343306d82fc916a21907de94901af Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 15 May 2026 11:09:47 -0500 Subject: [PATCH 066/222] eliminate string interpolation in command --- lib/datura/file_type.rb | 4 ++-- lib/datura/python/xslt_transform.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index e05fa4deb..73fece73d 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -197,12 +197,12 @@ def exec_xsl(input, xsl, ext, outpath=nil, params=nil) cmd = ["python3", python_script, "--input", input, "--xsl", xsl] if params params.each do |k, v| - cmd += ["--param", "#{k}=#{v}"] + cmd += ["--param", k.to_s, v.to_s] end end # append output path and base output URI for xsl:result-document secondary outputs if configured if outpath - cmd += ["--output", "#{outpath}/#{filename(false)}.#{ext}"] + cmd += ["--output", File.join(outpath, filename(false) + "." + ext)] cmd += ["--base-output-uri", outpath] end puts "using command #{cmd.inspect}" if @options["verbose"] diff --git a/lib/datura/python/xslt_transform.py b/lib/datura/python/xslt_transform.py index 24fc2a2ff..0195c75cd 100644 --- a/lib/datura/python/xslt_transform.py +++ b/lib/datura/python/xslt_transform.py @@ -11,8 +11,8 @@ def parse_args(): parser.add_argument("--input", required=True, help="Path to source XML file") parser.add_argument("--xsl", required=True, help="Path to XSL stylesheet") parser.add_argument("--output", required=False, help="Path to write output file") - parser.add_argument("--param", action="append", default=[], metavar="KEY=VALUE", - help="XSL parameter (repeatable)") + parser.add_argument("--param", action="append", default=[], nargs=2, metavar=("KEY", "VALUE"), + help="XSL parameter as separate key and value (repeatable)") parser.add_argument("--base-output-uri", required=False, dest="base_output_uri", help="Base URI for xsl:result-document secondary outputs (file:// URI or directory path)") # parse sys.argv, validate args, return namespace object with attributes @@ -26,11 +26,8 @@ def run_transform(input_path, xsl_path, params, output_path=None, base_output_ur with saxonche.PySaxonProcessor(license=False) as proc: # create XSLT 3.0 processor xslt_proc = proc.new_xslt30_processor() - # iterate list of kv strings - for kv in params: - if "=" not in kv: - raise ValueError(f"Invalid param format (expected KEY=VALUE): {kv!r}") - key, value = kv.split("=", 1) + # iterate list of [key, value] pairs + for key, value in params: xslt_proc.set_parameter(key, proc.make_string_value(value)) # parse and compile xsl file into executable executable = xslt_proc.compile_stylesheet(stylesheet_file=xsl_path) From 8aad8cfedf252264aa725e4db0d37513b84e4e89 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 15 May 2026 11:44:58 -0500 Subject: [PATCH 067/222] move python import and check for installed module before transforming files --- lib/datura/data_manager.rb | 12 ++++++++++++ lib/datura/python/xslt_transform.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 24f4898e9..0de8c5ef2 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -281,6 +281,14 @@ def set_up_logger ) end + def check_xslt_dependency + _, err, status = Open3.capture3("python3", "-c", "import saxonche") + unless status.success? + puts "saxonche Python module is not installed. Install it with: pip install saxonche\n#{err}".red + exit + end + end + def set_up_services if should_post?("es") # set up elasticsearch instance @@ -291,6 +299,10 @@ def set_up_services # set up posting URLs @solr_url = File.join(options["solr_path"], options["solr_core"], "update") end + + if should_transform?("html") || should_transform?("solr") + check_xslt_dependency + end end def should_post?(type) diff --git a/lib/datura/python/xslt_transform.py b/lib/datura/python/xslt_transform.py index 0195c75cd..3295d1aac 100644 --- a/lib/datura/python/xslt_transform.py +++ b/lib/datura/python/xslt_transform.py @@ -3,6 +3,14 @@ import sys from pathlib import Path +try: + import saxonche +# this should now be redundant with the addition of check_xslt_dependency to data_manager.rb +# but I am leaving it here for now as a backstop and in case we decide to return to this approach +# (the downside is that it will print an error for every call) +except ImportError as e: + print(f"XSLT transformation error: {e}", file=sys.stderr) + sys.exit(1) def parse_args(): # create the parser @@ -20,8 +28,6 @@ def parse_args(): def run_transform(input_path, xsl_path, params, output_path=None, base_output_uri=None): - # import here rather than at top so error is raised at call time with clear traceback if saxonche isn't installed - import saxonche # create Saxon processor using Home Edition tier with saxonche.PySaxonProcessor(license=False) as proc: # create XSLT 3.0 processor From 812b7b7314f923674b1f7e33280d55d602ccb45e Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 15 May 2026 14:08:35 -0500 Subject: [PATCH 068/222] add PyYAML --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9cb662327..0d6b8a149 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ omeka-s-tools==0.3.0 packaging==25.0 platformdirs==4.4.0 python-dotenv==1.1.1 +PyYAML==6.0.2 requests==2.32.5 requests-cache==1.2.1 typing_extensions==4.15.0 From 2851fbaf98a476b6224d3d04b7e4729fe1d094a6 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 15 May 2026 15:27:39 -0500 Subject: [PATCH 069/222] update gem versions --- Gemfile.lock | 16 +++++++--------- datura.gemspec | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8ef855c6e..a30898257 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,9 +2,8 @@ PATH remote: . specs: datura (1.1.0) - byebug (~> 11.0) - colorize (~> 0.8.1) - nokogiri (~> 1.10) + colorize (~> 1.0) + nokogiri (~> 1.18) pdf-reader (~> 2.12) rest-client (~> 2.1) @@ -15,7 +14,7 @@ GEM afm (0.2.2) bigdecimal (3.3.1) byebug (11.1.3) - colorize (0.8.1) + colorize (1.1.0) domain_name (0.6.20240107) hashery (2.1.2) http-accept (1.7.0) @@ -26,11 +25,9 @@ GEM logger mime-types-data (~> 3.2025, >= 3.2025.0507) mime-types-data (3.2026.0414) - mini_portile2 (2.8.9) minitest (5.27.0) netrc (0.11.0) - nokogiri (1.18.10) - mini_portile2 (~> 2.8.2) + nokogiri (1.18.10-arm64-darwin) racc (~> 1.4) nokogiri (1.18.10-x86_64-darwin) racc (~> 1.4) @@ -52,11 +49,12 @@ GEM bigdecimal (~> 3.1) PLATFORMS - ruby + arm64-darwin-24 x86_64-darwin-20 DEPENDENCIES - bundler (>= 1.16.0, < 3.0) + bundler (>= 2.0, < 5.0) + byebug (~> 11.0) datura! minitest (~> 5.0) rake (~> 13.0) diff --git a/datura.gemspec b/datura.gemspec index 86ba8785b..492818001 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -57,12 +57,12 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = "~> 3.1" - spec.add_runtime_dependency "colorize", "~> 0.8.1" - spec.add_runtime_dependency "nokogiri", "~> 1.10" + spec.add_runtime_dependency "colorize", "~> 1.0" + spec.add_runtime_dependency "nokogiri", "~> 1.18" spec.add_runtime_dependency "rest-client", "~> 2.1" spec.add_runtime_dependency "pdf-reader", "~> 2.12" spec.add_development_dependency "byebug", "~> 11.0" - spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" + spec.add_development_dependency "bundler", ">= 2.0", "< 5.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" end From 1151deb88ee7ba0174d2613f960cb4851e7d0488 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 11:41:47 -0500 Subject: [PATCH 070/222] add resume option to post --- lib/datura/parser_options/post.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index b6e48880f..4889d338c 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -49,6 +49,11 @@ def self.post_params options["regex"] = input end + options["resume"] = nil + opts.on('-R', '--resume [input]', 'Resume posting from (and including) the file matching this regex') do |input| + options["resume"] = input + end + options["transform_only"] = false opts.on('-t', '--transform-only', 'Do not post to solr / es') do options["transform_only"] = true From 33117c1def6a5e40d612ad0e7f88616e5931cfca Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 11:42:52 -0500 Subject: [PATCH 071/222] add resume_files function to sort and select --- lib/datura/helpers.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 671c1511c..da1467994 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -121,6 +121,30 @@ def self.regex_files(files, regex=nil) array end + # resume_files + # sorts files alphabetically and returns all files from the first file + # matching the resume regex onward (inclusive). Exits with an error if + # the regex matches zero or more than one file. + # params: files (array of file paths), regex (string) + # returns: array + def self.resume_files(files, regex) + sorted = files.sort_by { |f| File.basename(f, ".*") } + exp = Regexp.new(regex) + matches = sorted.select { |f| exp.match(File.basename(f, ".*")) } + + if matches.empty? + puts "ERROR: --resume regex '#{regex}' matched no files. Exiting.".red + exit 1 + elsif matches.length > 1 + names = matches.map { |f| File.basename(f, ".*") }.join(", ") + puts "ERROR: --resume regex '#{regex}' matched #{matches.length} files (#{names}). Refine your regex to match exactly one file. Exiting.".red + exit 1 + end + + resume_index = sorted.index(matches.first) + sorted[resume_index..] + end + # should_update? # determines if a user has changed a file since specified date # params: file (string path), since_date (Time format or nil) From cda844641d87bc90b9e0943cd957dcd6e3e7a3bc Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 11:43:57 -0500 Subject: [PATCH 072/222] add resumed option and adjust filtered var accordingly --- lib/datura/data_manager.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index ceca339d5..2cb342777 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -254,8 +254,14 @@ def prepare_files puts msg.yellow @log.warn(msg) end + # resume from (and including) a specific file + resumed = if @options["resume"] + Datura::Helpers.resume_files(regexed, @options["resume"]) + else + regexed + end # filter by date - filtered = regexed.select { |f| Datura::Helpers.should_update?(f, @options["update_time"]) } + filtered = resumed.select { |f| Datura::Helpers.should_update?(f, @options["update_time"]) } file_classes = [] @log.info("After filters (regex, update time), #{filtered.length}/#{files.length} files remaining") From 98f9a00c056c2002862abf800a59e7aadfcf00cf Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 11:44:22 -0500 Subject: [PATCH 073/222] update docs to add resume option --- docs/3_manage/post.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index 2a013f4c5..6cc766157 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -78,6 +78,16 @@ Transforms / posts only files matching a specific regular expression. DO NOT in Example: `post -r let0001` +```bash +-R, --resume [input] +``` + +Resume posting from (and including) the file matching this regex. Files are sorted alphabetically before the resume point is located. The regex must match exactly one file or the script will exit with an error. Can be combined with `-r` to resume within a filtered set. + +Example: `post -R let0050` (post all files from `let0050` onward) + +Example: `post -r let -R let0050` (post all `let` files from `let0050` onward) + ```bash -t, --transform-only ``` From 6269779742e48012b075e20b3d77eb949f7803b1 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 11:44:42 -0500 Subject: [PATCH 074/222] update helpers test to test resume option --- test/helpers_test.rb | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/helpers_test.rb b/test/helpers_test.rb index 0fa3ca13f..9291edbe5 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -105,6 +105,41 @@ def test_regex_files assert_equal 1, files.length end + def test_resume_files + test_files = %w[ + /path/to/cody.book.002.xml + /path/to/cat.let0001.xml + /path/to/cody.book.001.xml + /path/to/transmiss.mem.001.xml + /path/to/cody.news.001.xml + ] + + # exact match on one file: returns that file and all alphabetically after it + # alphabetical order: cat.let0001, cody.book.001, cody.book.002, cody.news.001, transmiss.mem.001 + files = Datura::Helpers.resume_files(test_files, "cody\.book\.002") + basenames = files.map { |f| File.basename(f, ".*") } + assert_equal %w[cody.book.002 cody.news.001 transmiss.mem.001], basenames + + # match on first file alphabetically: returns all files + files = Datura::Helpers.resume_files(test_files, "cat\.let0001") + assert_equal 5, files.length + + # match on last file: returns only that file + files = Datura::Helpers.resume_files(test_files, "transmiss\.mem\.001") + assert_equal 1, files.length + assert_equal "transmiss.mem.001", File.basename(files.first, ".*") + + # no match: exits with error + assert_raises(SystemExit) do + Datura::Helpers.resume_files(test_files, "zzz_no_such_file") + end + + # multiple matches: exits with error + assert_raises(SystemExit) do + Datura::Helpers.resume_files(test_files, "cody") + end + end + def test_should_update? hour_ago = Time.now - 60*60 test_file = "#{File.dirname(__FILE__)}/fixtures/should_update.txt" From 9879a4c97df9865bbaa82d2d7923ea8632eba8e7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:03:17 -0500 Subject: [PATCH 075/222] fix typo --- lib/datura/python/omeka_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 638bd9d1c..95860989e 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -271,7 +271,7 @@ def _load_config(path, env): return contents[env] - def __init__(self, config, env_config, environment, regex, media_skip, update_time=None)): + def __init__(self, config, env_config, environment, regex, media_skip, update_time=None): """ Initialise the context. Prefer OmekaContext.from_args() over calling this constructor directly except in tests. From 9444bc28cbc0ad27e83f858ba4ee6c166744ba81 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:03:50 -0500 Subject: [PATCH 076/222] add get_citation function to streamline fields --- lib/datura/python/field_definitions.py | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index f7a816cdd..b76f6a886 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -27,6 +27,10 @@ def __init__(self, omeka_data_base=""): # Stored as a private attribute and accessed only by uriData(). self._omeka_data_base = omeka_data_base + def _get_citation(self, json): + """Return the citation sub-dict, or {} if absent or null.""" + return json.get("citation") or {} + def title(self, json): return json.get("title", None) @@ -95,41 +99,41 @@ def relation(self, json): return relation_ids def publisher(self, json): - return (json.get("citation") or {}).get("publisher", None) + return self._get_citation(json).get("publisher", None) def biblID(self, json): #note: this field is not yet implemented in the schema - return (json.get("citation") or {}).get("id", None) + return self._get_citation(json).get("id", None) def biblTitle(self, json): - return (json.get("citation") or {}).get("title", None) + return self._get_citation(json).get("title", None) def biblPubPlace(self, json): - return (json.get("citation") or {}).get("pubplace", None) + return self._get_citation(json).get("pubplace", None) def issue(self, json): - return (json.get("citation") or {}).get("issue", None) + return self._get_citation(json).get("issue", None) def pageStart(self, json): - return (json.get("citation") or {}).get("page_start", None) + return self._get_citation(json).get("page_start", None) def pageEnd(self, json): - return (json.get("citation") or {}).get("page_end", None) + return self._get_citation(json).get("page_end", None) def section(self, json): - return (json.get("citation") or {}).get("section", None) + return self._get_citation(json).get("section", None) def volume(self, json): - return (json.get("citation") or {}).get("volume", None) + return self._get_citation(json).get("volume", None) def biblTitleA(self, json): - return (json.get("citation") or {}).get("title_a", None) + return self._get_citation(json).get("title_a", None) def biblTitleM(self, json): - return (json.get("citation") or {}).get("title_m", None) + return self._get_citation(json).get("title_m", None) def biblTitleJ(self, json): - return (json.get("citation") or {}).get("title_j", None) + return self._get_citation(json).get("title_j", None) def rightsHolder(self, json): return json.get("rights_holder", None) From e29c7a40b6c9acdc95868aec7d42225aa38c177b Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:10:26 -0500 Subject: [PATCH 077/222] add logger to top of module --- lib/datura/python/omeka.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 2fbbb4b71..dfd30718d 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -8,9 +8,11 @@ from datetime import datetime from pathlib import Path import json +import logging import os import re +logger = logging.getLogger(__name__) def add_media_to_item(ctx, item_id, media_file, payload=None, template_id=None, class_id=None): """ From d697fdf50576a9e1fb04b004cfe8c2da87108a26 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:11:23 -0500 Subject: [PATCH 078/222] delete duplicate function --- lib/datura/python/omeka.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index dfd30718d..9b9da9ef8 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -240,23 +240,6 @@ def prepare_property_value(value, property_id, label=""): return property_value -def filter_items(regex, pathlist): - """ - Filter a list of file paths to those matching a regex pattern. - - Used by both entrypoint scripts to restrict processing to a subset of - files when the -r / --regex flag is passed on the command line. - - Parameters: - * regex - regex pattern string, compiled with re.compile() - * pathlist - iterable of pathlib.Path or string paths to filter - - Returns a list containing only the paths whose string representation - matches the pattern. - """ - reg = re.compile(regex) - return [p for p in pathlist if reg.search(str(p))] - def filter_items(regex, pathlist): """ Filter a list of file paths to those matching a regex pattern. From 477992c02e48cdc133ad0c4a6485b0990398c00b Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:13:39 -0500 Subject: [PATCH 079/222] use context class for resource-item links --- lib/datura/python/omeka.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 9b9da9ef8..8d8112f0a 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -220,11 +220,13 @@ def prepare_property_value(value, property_id, label=""): } if data_type == 'resource:item': - # BUG: `self` is not defined here. This is dead code for current callers. - # Use ctx.client.prepare_property_value() for resource:item values. - property_value['@id'] = '{}/items/{}'.format(self.api_url, value['value']) # noqa: F821 - property_value['value_resource_id'] = value['value'] - property_value['value_resource_name'] = 'items' + # This branch is intentionally not implemented in this standalone function. + # Use ctx.client.prepare_property_value() for resource:item values instead — + # the library version has access to the API URL via the client instance. + raise NotImplementedError( + "resource:item values must use ctx.client.prepare_property_value(); " + "see link_item_record() in api_fields.py" + ) elif data_type == 'uri': property_value['@id'] = value['value'] # Fall back to the last URI segment when no explicit label is given. From 77ad8f55bb88369775d8c4dd8e33c0855cf47b74 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:27:14 -0500 Subject: [PATCH 080/222] add id_match to conditional to avoid AttributeError --- lib/datura/python/api_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 27d51466a..9a8c93f22 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -398,13 +398,13 @@ def get_matching_names_from_markdown(row, field): name_match = re.search(r"\[(.*?)\]", markdown_values) id_match = re.search(r"\]\((.*)\)", markdown_values) # Only collect the name if there is no associated identifier. - if name_match and not id_match.group(1): + if name_match and (not id_match or not id_match.group(1)): names.append(name_match.group(1)) else: for value in markdown_values: name_match = re.search(r"\[(.*?)\]", value) id_match = re.search(r"\]\((.*)\)", value) - if name_match and not id_match.group(1): + if name_match and (not id_match or not id_match.group(1)): names.append(name_match.group(1)) return names else: From a63a89cb31d79a63438aade2cefec9c458ca0927 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:30:44 -0500 Subject: [PATCH 081/222] make sure identifier exists before concatenation to avoid TypeError --- lib/datura/python/field_definitions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index b76f6a886..93e72f95c 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -240,7 +240,9 @@ def annotationsText(self, json): def itemText(self, json): text = json.get("text", None) if text and json.get("data_type"): - text += (" " + self.identifier(json)) + identifier = self.identifier(json) + if identifier: + text += (" " + identifier) return text def get_fields(omeka_data_base=""): From 8b9112a17f5a65f02e7efb6bcb9f72767dfe30da Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:33:09 -0500 Subject: [PATCH 082/222] add dict to maintain field ordering --- lib/datura/python/api_fields.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 9a8c93f22..58eec7238 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -285,9 +285,8 @@ def update_item_value(ctx, item, key, value, datatype="literal"): if type(value) in [str, int, float]: item = add_formatted_value(ctx, item, key, value, datatype) elif type(value) == list: - # Deduplicate and remove None entries before iterating. - value = list(set(value)) - value = [v for v in value if v is not None] + # Deduplicate (preserving insertion order) and remove None entries. + value = list(dict.fromkeys(v for v in value if v is not None)) for v in value: item = add_formatted_value(ctx, item, key, v, datatype) From f4f4a76d12325d46fb18a5a157d24c216a7d77d1 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:34:48 -0500 Subject: [PATCH 083/222] refactor source to match citation --- lib/datura/python/field_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 93e72f95c..d5fb490d9 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -178,7 +178,7 @@ def keyword5(self, json): return json.get("keywords5", None) def source(self, json): - return json.get("has_source") and json.get("has_source", {}).get("title") + return (json.get("has_source") or {}).get("title") def medium(self, json): return json.get("medium", None) From 79f1f2f87de9b50541cb5ef736725cc408a0a0f1 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:36:22 -0500 Subject: [PATCH 084/222] add timeout for iiif request --- lib/datura/python/html_and_media_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index deff3ff05..b4ae2a699 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -207,7 +207,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): # --- Download --- try: logger.info("Downloading thumbnail for %r", identifier) - response = requests.get(thumbnail_remote) + response = requests.get(thumbnail_remote, timeout=30) response.raise_for_status() with open(thumbnail_local, "wb") as thumb_file: thumb_file.write(response.content) From fc5e2e6de8ecdcd7e8eefcdd7a835c8d2d281051 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:38:00 -0500 Subject: [PATCH 085/222] specify encoding for html files --- lib/datura/python/html_and_media_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index b4ae2a699..4703cbf8a 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -278,7 +278,7 @@ def ingest_html(ctx, json_item, matching_item, html_dir): file_path = html_dir / "{}.html".format(identifier) try: - with open(file_path, "r") as file: + with open(file_path, "r", encoding="utf-8") as file: html_content = file.read() except FileNotFoundError: # A missing HTML file is common for items that have no text From 3c2368f6ff2bacd30f0173d249181700b9271344 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:39:23 -0500 Subject: [PATCH 086/222] add error handling to dateYear --- lib/datura/python/field_definitions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index d5fb490d9..19604a08c 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -81,8 +81,10 @@ def date(self, json): def dateYear(self, json): date_to_parse = json.get("date", None) if date_to_parse: - year = datetime.strptime(date_to_parse, "%Y-%m-%d").year - return year + try: + return datetime.strptime(date_to_parse, "%Y-%m-%d").year + except ValueError: + return None def dateDisplay(self, json): return json.get("date_display", None) From e202ca0bc1430c5910d8ca579fe190d5a86e79cd Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:41:27 -0500 Subject: [PATCH 087/222] add error handling for csv ingest --- lib/datura/python/api_fields.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 58eec7238..d5f49b636 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -247,7 +247,11 @@ def get_json_value(row, name): if len(row[name]) > 0: if row[name].startswith('["'): # Deserialise a JSON-encoded array. - return json.loads(row[name]) + try: + return json.loads(row[name]) + except json.JSONDecodeError: + logger.warning("Could not parse JSON value for field %r: %r", name, row[name]) + return row[name] elif ";;;" in row[name]: # Split a semicolon-delimited multi-value string. return row[name].split(";;;") From 31ea95ee5937b95059e7735415a71d66e3d94906 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:47:07 -0500 Subject: [PATCH 088/222] use logger instead of print --- lib/datura/python/omeka.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 8d8112f0a..8c9843f06 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -130,7 +130,7 @@ def prepare_item_payload_using_template(ctx, terms, template_id): if term not in template_properties: # Terms outside the template are intentionally dropped — each # collection defines which fields are relevant to its template. - print('Term {} not in template'.format(term)) + logger.warning("Term %r not in template; skipping", term) continue property_details = template_properties[term] @@ -143,9 +143,9 @@ def prepare_item_payload_using_template(ctx, terms, template_id): # Validate the supplied data type against the template's allowed types. if 'type' in value and value['type'] not in property_details['type']: - print( - 'Data type "{}" for term "{}" not allowed by template' - .format(value['type'], term) + logger.warning( + "Data type %r for term %r not allowed by template; skipping value", + value['type'], term ) break @@ -159,7 +159,7 @@ def prepare_item_payload_using_template(ctx, terms, template_id): value['type'] = 'literal' else: # Cannot determine a type; skip this value. - print('Specify data type for term "{}"'.format(term)) + logger.warning("Cannot determine data type for term %r; skipping value",term) break if "property_id" in value: From b524302898f38064604beeeafd6ed7d9fdd481ce Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 21 May 2026 14:47:54 -0500 Subject: [PATCH 089/222] use isinstance instead of type --- lib/datura/python/api_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index d5f49b636..24bfc22da 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -286,9 +286,9 @@ def update_item_value(ctx, item, key, value, datatype="literal"): # carried forward when the source data no longer has a value for this field. item[key] = [] - if type(value) in [str, int, float]: + if isinstance(value, (str, int, float)): item = add_formatted_value(ctx, item, key, value, datatype) - elif type(value) == list: + elif isinstance(value, list): # Deduplicate (preserving insertion order) and remove None entries. value = list(dict.fromkeys(v for v in value if v is not None)) for v in value: From c10050b88337c2e73b6799ee16eaf9a200c5bfa4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 11:40:48 -0500 Subject: [PATCH 090/222] refactor resume as proceed, validate and rescue regex --- docs/3_manage/post.md | 16 ++++++++-------- lib/datura/data_manager.rb | 8 ++++---- lib/datura/helpers.rb | 29 ++++++++++++++++++++--------- lib/datura/parser_options/post.rb | 8 ++++++++ test/helpers_test.rb | 22 ++++++++++++++++------ 5 files changed, 56 insertions(+), 27 deletions(-) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index 5607d97d1..b30bbbae4 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -79,22 +79,22 @@ Solr specific, this will post documents but will not "commit" them to the index. Outputs transformed files to a collection's `output/[environment]/[type]`. This is useful for debugging and inspection purposes. ```bash --r, --regex [input] +-p, --proceed [input] ``` -Transforms / posts only files matching a specific regular expression. DO NOT include the file extension. +Proceed with posting from (and including) the file matching this regex. Files are sorted alphabetically before the proceed point is located. The regex must match exactly one file or the script will exit with an error. Can be combined with `-r` to proceed within a filtered set. -Example: `post -r let0001` +Example: `post -R let0050` (post all files from `let0050` onward) + +Example: `post -r let -R let0050` (post all `let` files from `let0050` onward) ```bash --R, --resume [input] +-r, --regex [input] ``` -Resume posting from (and including) the file matching this regex. Files are sorted alphabetically before the resume point is located. The regex must match exactly one file or the script will exit with an error. Can be combined with `-r` to resume within a filtered set. - -Example: `post -R let0050` (post all files from `let0050` onward) +Transforms / posts only files matching a specific regular expression. DO NOT include the file extension. -Example: `post -r let -R let0050` (post all `let` files from `let0050` onward) +Example: `post -r let0001` ```bash -t, --transform-only diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 2cb342777..931f0adf3 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -254,14 +254,14 @@ def prepare_files puts msg.yellow @log.warn(msg) end - # resume from (and including) a specific file - resumed = if @options["resume"] - Datura::Helpers.resume_files(regexed, @options["resume"]) + # prcoeed from (and including) a specific file + proceeded = if @options["resume"] + Datura::Helpers.proceeded_files(regexed, @options["proceed"]) else regexed end # filter by date - filtered = resumed.select { |f| Datura::Helpers.should_update?(f, @options["update_time"]) } + filtered = proceeded.select { |f| Datura::Helpers.should_update?(f, @options["update_time"]) } file_classes = [] @log.info("After filters (regex, update time), #{filtered.length}/#{files.length} files remaining") diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index da1467994..1a4227448 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -111,7 +111,7 @@ def self.normalize_space(abnormal) def self.regex_files(files, regex=nil) array = files.nil? ? [] : files if !files.nil? && !regex.nil? - exp = Regexp.new(regex) + exp = validate_regex(regex, "--regex") array = files.select do |file| file_name = File.basename(file, ".*") match = exp.match(file_name) @@ -121,28 +121,28 @@ def self.regex_files(files, regex=nil) array end - # resume_files + # proceed_files # sorts files alphabetically and returns all files from the first file - # matching the resume regex onward (inclusive). Exits with an error if + # matching the proceed regex onward (inclusive). Exits with an error if # the regex matches zero or more than one file. # params: files (array of file paths), regex (string) # returns: array - def self.resume_files(files, regex) + def self.proceed_files(files, regex) sorted = files.sort_by { |f| File.basename(f, ".*") } - exp = Regexp.new(regex) + exp = validate_regex(regex, "--proceed") matches = sorted.select { |f| exp.match(File.basename(f, ".*")) } if matches.empty? - puts "ERROR: --resume regex '#{regex}' matched no files. Exiting.".red + puts "ERROR: --proceed regex '#{regex}' matched no files. Exiting.".red exit 1 elsif matches.length > 1 names = matches.map { |f| File.basename(f, ".*") }.join(", ") - puts "ERROR: --resume regex '#{regex}' matched #{matches.length} files (#{names}). Refine your regex to match exactly one file. Exiting.".red + puts "ERROR: --proceed regex '#{regex}' matched #{matches.length} files (#{names}). Refine your regex to match exactly one file. Exiting.".red exit 1 end - resume_index = sorted.index(matches.first) - sorted[resume_index..] + proceed_index = sorted.index(matches.first) + sorted[proceed_index..] end # should_update? @@ -160,6 +160,17 @@ def self.should_update?(file, since_date=nil) end end + # validate_regex + # compiles a regex string; prints a readable error and exits if invalid + # params: regex (string), flag (string, e.g. "--regex" or "--proceed") + # returns: Regexp + def self.validate_regex(regex, flag) + Regexp.new(regex) + rescue RegexpError => e + puts "ERROR: Invalid regex for #{flag} '#{regex}': #{e.message}".red + exit 1 + end + def self.construct_auth_header(options) username = options["es_user"] password = options["es_password"] diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index ef938751e..243b8d227 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -44,11 +44,17 @@ def self.post_params options["output"] = true end + options["proceed"] = nil + opts.on('-p', '--proceed [input]', 'Proceed with posting from (and including) the file matching this regex') do |input| + options["proceed"] = input + end + options["regex"] = nil opts.on('-r', '--regex [input]', 'Only post files matching this regex') do |input| options["regex"] = input end +<<<<<<< HEAD options["resume"] = nil opts.on('-R', '--resume [input]', 'Resume posting from (and including) the file matching this regex') do |input| options["resume"] = input @@ -59,6 +65,8 @@ def self.post_params options["csv_rows"] = input end +======= +>>>>>>> 1da6adb2d (rename resume as proceed, with -p flag, to avoid confusion) options["transform_only"] = false opts.on('-t', '--transform-only', 'Do not post to solr / es') do options["transform_only"] = true diff --git a/test/helpers_test.rb b/test/helpers_test.rb index 9291edbe5..0a2362948 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -103,9 +103,14 @@ def test_regex_files # return a specific id files = Datura::Helpers.regex_files(test_files, "cat.let0001") assert_equal 1, files.length + + # invalid regex: exits with error + assert_raises(SystemExit) do + Datura::Helpers.regex_files(test_files, "[unclosed") + end end - def test_resume_files + def test_proceed_files test_files = %w[ /path/to/cody.book.002.xml /path/to/cat.let0001.xml @@ -116,27 +121,32 @@ def test_resume_files # exact match on one file: returns that file and all alphabetically after it # alphabetical order: cat.let0001, cody.book.001, cody.book.002, cody.news.001, transmiss.mem.001 - files = Datura::Helpers.resume_files(test_files, "cody\.book\.002") + files = Datura::Helpers.proceed_files(test_files, "cody\.book\.002") basenames = files.map { |f| File.basename(f, ".*") } assert_equal %w[cody.book.002 cody.news.001 transmiss.mem.001], basenames # match on first file alphabetically: returns all files - files = Datura::Helpers.resume_files(test_files, "cat\.let0001") + files = Datura::Helpers.proceed_files(test_files, "cat\.let0001") assert_equal 5, files.length # match on last file: returns only that file - files = Datura::Helpers.resume_files(test_files, "transmiss\.mem\.001") + files = Datura::Helpers.proceed_files(test_files, "transmiss\.mem\.001") assert_equal 1, files.length assert_equal "transmiss.mem.001", File.basename(files.first, ".*") # no match: exits with error assert_raises(SystemExit) do - Datura::Helpers.resume_files(test_files, "zzz_no_such_file") + Datura::Helpers.proceed_files(test_files, "zzz_no_such_file") end # multiple matches: exits with error assert_raises(SystemExit) do - Datura::Helpers.resume_files(test_files, "cody") + Datura::Helpers.proceed_files(test_files, "cody") + end + + # invalid regex: exits with error + assert_raises(SystemExit) do + Datura::Helpers.proceed_files(test_files, "[unclosed") end end From 2f9c4733b7b833ea57a1552bce7f7552f7fa414f Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 11:43:22 -0500 Subject: [PATCH 091/222] remove half-finished conflict resolution --- lib/datura/parser_options/post.rb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index 243b8d227..e97f15d24 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -54,19 +54,6 @@ def self.post_params options["regex"] = input end -<<<<<<< HEAD - options["resume"] = nil - opts.on('-R', '--resume [input]', 'Resume posting from (and including) the file matching this regex') do |input| - options["resume"] = input - - options["csv_rows"] = nil - opts.on('-c', '--csv-rows [input]', - 'Only process CSV rows whose identifier (id/identifier column) matches this regex.') do |input| - options["csv_rows"] = input - end - -======= ->>>>>>> 1da6adb2d (rename resume as proceed, with -p flag, to avoid confusion) options["transform_only"] = false opts.on('-t', '--transform-only', 'Do not post to solr / es') do options["transform_only"] = true From f46a2b2c30db196054a00b51798624e2d054e73e Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 11:53:20 -0500 Subject: [PATCH 092/222] move csv rows option to correct alphabetical location --- lib/datura/parser_options/post.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index e97f15d24..2cf48bdc8 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -39,6 +39,12 @@ def self.post_params options["commit"] = false end + options["csv_rows"] = nil + opts.on('-c', '--csv-rows [input]', + 'Only process CSV rows whose identifier (id/identifier column) matches this regex.') do |input| + options["csv_rows"] = input + end + options["output"] = false opts.on('-o', '--output', 'Write solr and elasticsearch docs to file') do options["output"] = true From ee0af0c9046dec398cbda632723f574da8c7492c Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 12:08:04 -0500 Subject: [PATCH 093/222] add checkpoint, fix typo --- docs/3_manage/post.md | 10 ++++++++++ lib/datura/data_manager.rb | 34 +++++++++++++++++++++++++++++++++- lib/datura/helpers.rb | 28 ++++++++++++++++++++++++++++ test/helpers_test.rb | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 1 deletion(-) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index b30bbbae4..f9b2dd4b8 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -88,6 +88,16 @@ Example: `post -R let0050` (post all files from `let0050` onward) Example: `post -r let -R let0050` (post all `let` files from `let0050` onward) +**Checkpoint file**: After each batch of files is posted (not in `--transform-only` mode), Datura writes the last posted filename to `logs/proceed_{environment}` in your collection directory. Add `logs/proceed_*` to your collection's `.gitignore` to avoid committing this file. + +**Interactive resume**: When `-p` is given with no value (`post -p`), Datura reads the checkpoint file and prompts: + +``` +Continue from ? (y/n): +``` + +Enter `y` to resume posting from that file, or `n` to exit. If no checkpoint file exists yet, the script exits with an error directing you to run `post` at least once first. + ```bash -r, --regex [input] ``` diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 931f0adf3..e02cec435 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -80,6 +80,7 @@ def run @log.info(msg) puts msg pre_file_preparation + handle_proceed_prompt @files = prepare_files pre_batch_processing batch_process_files @@ -128,6 +129,10 @@ def batch_process_files end # wait for all the files to process before moving on with the next chunk threads.each { |t| t.join } + # save checkpoint after chunk completes (not in transform-only mode) + unless @options["transform_only"] + Datura::Helpers.write_checkpoint(files_subset.last.filename(false), @options) + end end end @@ -206,6 +211,33 @@ def get_files files end + def handle_proceed_prompt + # Only act when -p was given with no value (proceed is nil, not false) + return unless @options["proceed"].nil? + + checkpoint = Datura::Helpers.read_checkpoint(@options) + if checkpoint.nil? + path = Datura::Helpers.checkpoint_path(@options) + msg = "ERROR: --proceed given with no value but no checkpoint file found at #{path}. Run post at least once without -p to create a checkpoint.".red + puts msg + @log.error(msg) + exit 1 + end + + print "Continue from #{checkpoint}? (y/n): " + STDOUT.flush + response = STDIN.gets&.chomp&.downcase || "" + if response == "y" + @options["proceed"] = checkpoint + msg = "Resuming from checkpoint: #{checkpoint}" + puts msg + @log.info(msg) + else + puts "Exiting." + exit 0 + end + end + def options_msg msg = "Start Time: #{Time.now}\n" msg << "Running script with following options:\n" @@ -256,7 +288,7 @@ def prepare_files end # prcoeed from (and including) a specific file proceeded = if @options["resume"] - Datura::Helpers.proceeded_files(regexed, @options["proceed"]) + Datura::Helpers.proceed_files(regexed, @options["proceed"]) else regexed end diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 1a4227448..d35904384 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -145,6 +145,34 @@ def self.proceed_files(files, regex) sorted[proceed_index..] end + # checkpoint_path + # returns the full path to the proceed checkpoint file + # params: options (hash with "collection_dir" and "environment" keys) + # returns: string + def self.checkpoint_path(options) + File.join(options["collection_dir"], "logs", "proceed_#{options["environment"]}") + end + + # read_checkpoint + # reads the proceed checkpoint file and returns its contents + # params: options (hash) + # returns: string (basename without extension) or nil if file missing/empty + def self.read_checkpoint(options) + path = checkpoint_path(options) + return nil unless File.exist?(path) + content = File.read(path).strip + content.empty? ? nil : content + end + + # write_checkpoint + # writes the basename of the last posted file to the checkpoint file + # params: basename (string, filename without extension), options (hash) + # returns: nil + def self.write_checkpoint(basename, options) + path = checkpoint_path(options) + File.write(path, "#{basename}\n") + end + # should_update? # determines if a user has changed a file since specified date # params: file (string path), since_date (Time format or nil) diff --git a/test/helpers_test.rb b/test/helpers_test.rb index 0a2362948..3fa51c168 100644 --- a/test/helpers_test.rb +++ b/test/helpers_test.rb @@ -150,6 +150,39 @@ def test_proceed_files end end + def test_checkpoint_helpers + Dir.mktmpdir do |tmpdir| + opts = { + "collection_dir" => tmpdir, + "environment" => "test" + } + log_dir = File.join(tmpdir, "logs") + FileUtils.mkdir_p(log_dir) + expected_path = File.join(log_dir, "proceed_test") + + assert_equal expected_path, Datura::Helpers.checkpoint_path(opts) + + # returns nil when file does not exist + assert_nil Datura::Helpers.read_checkpoint(opts) + + # creates file with correct content + Datura::Helpers.write_checkpoint("let0050", opts) + assert File.exist?(expected_path) + assert_equal "let0050", File.read(expected_path).strip + + # returns the stored basename + assert_equal "let0050", Datura::Helpers.read_checkpoint(opts) + + # overwrites on second write + Datura::Helpers.write_checkpoint("let0100", opts) + assert_equal "let0100", Datura::Helpers.read_checkpoint(opts) + + # returns nil for empty/whitespace file + File.write(expected_path, " \n") + assert_nil Datura::Helpers.read_checkpoint(opts) + end + end + def test_should_update? hour_ago = Time.now - 60*60 test_file = "#{File.dirname(__FILE__)}/fixtures/should_update.txt" From b1eabe8cad812ea936e35966ec57a66f067c7424 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 12:21:55 -0500 Subject: [PATCH 094/222] update overlooked resume and change proceed default to false --- lib/datura/data_manager.rb | 2 +- lib/datura/parser_options/post.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index e02cec435..03d4cbd20 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -287,7 +287,7 @@ def prepare_files @log.warn(msg) end # prcoeed from (and including) a specific file - proceeded = if @options["resume"] + proceeded = if @options["proceed"] Datura::Helpers.proceed_files(regexed, @options["proceed"]) else regexed diff --git a/lib/datura/parser_options/post.rb b/lib/datura/parser_options/post.rb index 2cf48bdc8..29f47c343 100644 --- a/lib/datura/parser_options/post.rb +++ b/lib/datura/parser_options/post.rb @@ -50,7 +50,7 @@ def self.post_params options["output"] = true end - options["proceed"] = nil + options["proceed"] = false opts.on('-p', '--proceed [input]', 'Proceed with posting from (and including) the file matching this regex') do |input| options["proceed"] = input end From 854e5ac88f0752d3d4d692c47a39920081090683 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 16:31:27 -0500 Subject: [PATCH 095/222] add iiif_collection param to context class --- lib/datura/python/omeka_context.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 95860989e..dad4cb009 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -321,6 +321,8 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti self.omeka_data_base = config["omeka_data_base"] # iiif_server is optional — not all collections ingest thumbnails. self.iiif_server = config.get("iiif_server", "") + # iiif_collection is optional — not all collections have different iiif collection names. + self.iiif_collection = config.get("iiif_collection", "") # Keep the environment-specific dict for the item_set_id property. self._env_config = env_config From e101a10fd26ca89b9c157ef469af74246f1e2fa3 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 28 May 2026 16:41:17 -0500 Subject: [PATCH 096/222] use iiif_collection for collection_name in thumbnail function if it exists --- lib/datura/python/html_and_media_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 4703cbf8a..f46d3d4f1 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -181,7 +181,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): * iiif_dir - pathlib.Path pointing to the local IIIF output directory where the downloaded thumbnail is cached temporarily """ - collection_name = json_item.get("collection", "") + collection_name = ctx.iiif_collection if ctx.iiif_collection else json_item.get("collection", "") cover_image = json_item.get("cover_image") identifier = json_item.get("identifier", "unknown") From 5fa5621bc6f34cdb2591728f3b9c1f9462c38aab Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 10:19:32 -0500 Subject: [PATCH 097/222] make development default environment --- bin/post_omeka | 2 +- bin/post_omeka_html | 2 +- lib/datura/python/html_and_media_ingest.py | 12 ++++++------ lib/datura/python/json_to_omeka.py | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index 4fe60b92f..f6cf4b23b 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -16,7 +16,7 @@ optparse = OptionParser.new do |opts| generate_es = false end - opts.on('-e', '--environment [input]', 'environment (development, production)') do |input| + opts.on('-e', '--environment [input]', 'environment (default: development, or production)') do |input| if input && input.length > 0 options["environment"] = input end diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 7955d2b02..0d46b9d83 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -16,7 +16,7 @@ optparse = OptionParser.new do |opts| options["media_skip"] = true end - opts.on('-e', '--environment [input]', 'environment (development, production)') do |input| + opts.on('-e', '--environment [input]', 'environment (default: development, or production)') do |input| if input && input.length > 0 options["environment"] = input end diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index f46d3d4f1..7b53e0ae1 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -15,10 +15,10 @@ as an HTML media object. Usage (from collection root directory): - python3 html_and_media_ingest.py -e development + python3 html_and_media_ingest.py # defaults to development python3 html_and_media_ingest.py -e production -r "some_pattern" - python3 html_and_media_ingest.py -e development -m # skip items that already have media - python3 html_and_media_ingest.py -e development --log-level DEBUG + python3 html_and_media_ingest.py -m # skip items that already have media + python3 html_and_media_ingest.py --log-level DEBUG The script is invoked by bin/post_omeka_html in the Datura gem. The Ruby wrapper passes -e, -r, and -m arguments from its own CLI. @@ -58,7 +58,7 @@ def _parse_args(): Parse command-line arguments for the HTML/media ingest entrypoint. Returns an argparse.Namespace with: - * environment - "development" or "production" + * environment - "development" or "production" (default: "development") * regex - optional file-filter pattern string, or None * media_skip - bool; True skips items that already have 2+ media objects * log_level - logging level string, default "INFO" @@ -68,8 +68,8 @@ def _parse_args(): ) parser.add_argument( "-e", "--environment", - required=True, - help="Target environment: 'development' or 'production'.", + default="development", + help="Target environment: 'development' or 'production' (default: development).", ) parser.add_argument( "-r", "--regex", diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 6e9816a63..ddcb69f6c 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -19,9 +19,9 @@ before they can be linked to one another. Usage (from collection root directory): - python3 json_to_omeka.py -e development + python3 json_to_omeka.py # defaults to development python3 json_to_omeka.py -e production -r "some_pattern" - python3 json_to_omeka.py -e development --log-level DEBUG + python3 json_to_omeka.py --log-level DEBUG The script is invoked by bin/post_omeka in the Datura gem. The Ruby wrapper passes -e and -r arguments from its own CLI. No changes to bin/post_omeka @@ -62,7 +62,7 @@ def _parse_args(): Parse command-line arguments for the JSON-to-Omeka entrypoint. Returns an argparse.Namespace with: - * environment - "development" or "production" + * environment - "development" or "production" (default: "development") * regex - optional file-filter pattern string, or None * log_level - logging level string, default "INFO" @@ -76,8 +76,8 @@ def _parse_args(): ) parser.add_argument( "-e", "--environment", - required=True, - help="Target environment: 'development' or 'production'.", + default="development", + help="Target environment: 'development' or 'production' (default: development).", ) parser.add_argument( "-r", "--regex", From f05b1caaef7452f946a9016f2ebb255aea7d080d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 10:34:08 -0500 Subject: [PATCH 098/222] add extension check for images --- lib/datura/python/html_and_media_ingest.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 7b53e0ae1..17d386e3e 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -27,6 +27,7 @@ import argparse import json import logging +import os import sys from pathlib import Path @@ -190,19 +191,30 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): logger.debug("No cover_image for %r; skipping thumbnail ingest", identifier) return + # Parse any existing extension from the cover_image name. Image identifiers + # often contain dots that are not extensions (e.g. loc.00001, ccda.let00001), + # so only treat the suffix as an extension if it is a known image format. + _KNOWN_IMAGE_EXTS = {".jpg", ".jpeg", ".png"} + _stem, _ext = os.path.splitext(cover_image) + if _ext.lower() in _KNOWN_IMAGE_EXTS: + stem, image_ext = _stem, _ext + else: + stem, image_ext = cover_image, ".jpg" + # Construct the IIIF Image API URL for the thumbnail. # The !200,200 size specifier requests a thumbnail that fits within a # 200×200 bounding box while preserving aspect ratio. thumbnail_remote = ( - "{}/iiif/2/{collection}%2F{image}.jpg/full/!200,200/0/default.jpg".format( + "{}/iiif/2/{collection}%2F{image}{ext}/full/!200,200/0/default.jpg".format( ctx.iiif_server, collection=collection_name, - image=cover_image, + image=stem, + ext=image_ext, ) ) # Cache the thumbnail locally using the same URL-encoded filename so that # re-runs can be inspected on disk if needed. - thumbnail_local = iiif_dir / "{}%2F{}.jpg".format(collection_name, cover_image) + thumbnail_local = iiif_dir / "{}%2F{}{}".format(collection_name, stem, image_ext) # --- Download --- try: From 3961c79eae7a0f3cde918ee58954eab65a16e9ca Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 11:14:17 -0500 Subject: [PATCH 099/222] remove call to datamanager for non-existent check of omeka config values --- bin/post_omeka | 3 +-- bin/post_omeka_html | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index f6cf4b23b..cfa279334 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -36,9 +36,8 @@ end optparse.parse(ARGV) #add options to output a json file instead of posting it to Elasticsearch ARGV.unshift("-x", "es", "-o", "-t") -#create and validate DataManager before conditional run +#create DataManager before conditional run manager = Datura::DataManager.new -manager.check_omeka_options #exit with clear error if any key is missing #skip generation step with option -s if generate_es manager.run diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 0d46b9d83..9d81cd030 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -38,9 +38,8 @@ optparse.parse(ARGV) ARGV.delete("-m") #add option to generate html ARGV.unshift("-x", "html") -#create and validate DataManager before conditional run +#create DataManager before conditional run manager = Datura::DataManager.new -manager.check_omeka_options #exit with clear error if any key is missing #skip generation step with option -s if generate_es manager.run From 08e61e17306295ea04271a02c791fa2b5a912f52 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 15:43:29 -0500 Subject: [PATCH 100/222] fix -s option by deleting before manager invocation --- bin/post_omeka | 3 +++ bin/post_omeka_html | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/post_omeka b/bin/post_omeka index cfa279334..1602aea70 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -34,6 +34,9 @@ optparse = OptionParser.new do |opts| end #parse, but do not consume, command line arguments (the usual parse! would consume them) optparse.parse(ARGV) +#remove options not used in the main script +ARGV.delete("-s") +ARGV.delete("--skip") #add options to output a json file instead of posting it to Elasticsearch ARGV.unshift("-x", "es", "-o", "-t") #create DataManager before conditional run diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 9d81cd030..3e2d44a29 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -34,8 +34,11 @@ optparse = OptionParser.new do |opts| end #parse, but do not consume, command line arguments (the usual parse! would consume them) optparse.parse(ARGV) -#remove option not used in the main script +#remove options not used in the main script ARGV.delete("-m") +ARGV.delete("--media_skip") +ARGV.delete("-s") +ARGV.delete("--skip") #add option to generate html ARGV.unshift("-x", "html") #create DataManager before conditional run From b85ab6b5b5539acb858242c479240c6050822cf6 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 15:45:47 -0500 Subject: [PATCH 101/222] display warning for missing identifier --- lib/datura/python/html_and_media_ingest.py | 2 +- lib/datura/python/json_to_omeka.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 17d386e3e..1564467c6 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -358,7 +358,7 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): for json_item in json_items: identifier = json_item.get("identifier") if not identifier: - logger.debug("Skipping item without identifier in %s", filename) + logger.warning("Skipping item without identifier in %s", filename) continue # --- Look up the item in Omeka --- diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index ddcb69f6c..58a72b3f5 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -122,7 +122,7 @@ def post_items(ctx, pathlist): * Items that return multiple Omeka matches are skipped with a warning (duplicate identifiers indicate a data integrity problem that must be resolved in the Omeka admin UI before the item can be re-posted). - * Items whose identifier field is falsy are skipped silently. + * Items whose identifier field is falsy are skipped with a warning. Per-item errors (API failures, malformed payload) are recorded via ctx.record_error() and do NOT halt the run. Fatal errors (wrong @@ -145,8 +145,7 @@ def post_items(ctx, pathlist): identifier = json_item.get("identifier") if not identifier: # Records without an identifier cannot be matched or created. - # This is expected for some document types; log at DEBUG only. - logger.debug("Skipping item without identifier in %s", filename) + logger.warning("Skipping item without identifier in %s", filename) continue try: From dc2109e6b2c6b0cef79da5ff80399ea110d6d971 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 16:15:47 -0500 Subject: [PATCH 102/222] add finsh_run to print errors and runtime for omeka post --- bin/post_omeka | 12 +++++++++++ bin/post_omeka_html | 12 +++++++++++ lib/datura/python/html_and_media_ingest.py | 11 +++++++++-- lib/datura/python/json_to_omeka.py | 15 ++++++++------ lib/datura/python/omeka_context.py | 23 ++++++++++++++++++++++ 5 files changed, 65 insertions(+), 8 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index 1602aea70..c90073f76 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -3,6 +3,7 @@ require "datura" require "optparse" require "shellwords" +require "tempfile" @usage = "Usage: post_omeka -[options]..." @@ -50,6 +51,9 @@ datura_dir = File.join(File.dirname(__FILE__), "..") python_script_path = File.join(datura_dir, "lib", "datura", "python", "json_to_omeka.py") #run posting script into Omeka S if File.exist?("#{python_script_path}") + error_file = Tempfile.new(["omeka_errors", ".txt"]) + error_file_path = error_file.path + error_file.close command = ["python3", python_script_path] if options["environment"] command.append("-e", Shellwords.escape(options["environment"])) @@ -57,7 +61,15 @@ if File.exist?("#{python_script_path}") if options["regex"] command.append("-r", Shellwords.escape(options["regex"])) end + command.append("--error-file", error_file_path) system(*command) + omeka_errors = begin + Integer(File.read(error_file_path).strip) + rescue + 0 + end + File.unlink(error_file_path) rescue nil + puts "#{omeka_errors} Omeka posting error(s)" else puts("Omeka script not found at #{python_script_path}".red) end \ No newline at end of file diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 3e2d44a29..c7fb89c2a 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -2,6 +2,7 @@ require "datura" require "shellwords" +require "tempfile" generate_es = true options = {} @@ -53,6 +54,9 @@ datura_dir = File.join(File.dirname(__FILE__), "..") python_script_path = File.join(datura_dir, "lib", "datura", "python", "html_and_media_ingest.py") #run posting script into Omeka S if File.exist?("#{python_script_path}") + error_file = Tempfile.new(["omeka_errors", ".txt"]) + error_file_path = error_file.path + error_file.close command = ["python3", python_script_path] if options["environment"] command.append("-e", Shellwords.escape(options["environment"])) @@ -63,7 +67,15 @@ if File.exist?("#{python_script_path}") if options["media_skip"] command.append("-m") end + command.append("--error-file", error_file_path) system(*command) + omeka_errors = begin + Integer(File.read(error_file_path).strip) + rescue + 0 + end + File.unlink(error_file_path) rescue nil + puts "#{omeka_errors} Omeka posting error(s)" else puts("Omeka script not found at #{python_script_path}".red) end \ No newline at end of file diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 1564467c6..4c6a54790 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -43,6 +43,7 @@ OmekaMultipleMatchesError, OmekaItemNotFoundError, configure_logging, + finish_run, ) # Module-level logger. Records from this module appear as @@ -109,6 +110,12 @@ def _parse_args(): dest="log_level", help="Set the logging verbosity (default: INFO).", ) + parser.add_argument( + "--error-file", + dest="error_file", + default=None, + help="If provided, write the integer error count to this file before exiting.", + ) return parser.parse_args() @@ -432,6 +439,7 @@ def main(): 7. Report errors; exit 1 if any failures, 0 if clean. """ args = _parse_args() + start_time = time.time() configure_logging(args.log_level) # OmekaConfigError propagates here as a fatal error — missing or broken @@ -462,8 +470,7 @@ def main(): process_items(ctx, pathlist, html_dir, iiif_dir) - ctx.report_errors() - sys.exit(1 if ctx._errors else 0) + finish_run(ctx, args, start_time) if __name__ == "__main__": diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 58a72b3f5..3d2f73b20 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -45,6 +45,7 @@ OmekaItemNotFoundError, OmekaMultipleMatchesError, configure_logging, + finish_run, ) from omeka import filter_items, filter_items_by_date, prepare_item_payload_using_template @@ -105,6 +106,12 @@ def _parse_args(): dest="log_level", help="Set the logging verbosity (default: INFO).", ) + parser.add_argument( + "--error-file", + dest="error_file", + default=None, + help="If provided, write the integer error count to this file before exiting.", + ) return parser.parse_args() @@ -400,6 +407,7 @@ def main(): 0 if all items succeeded. """ args = _parse_args() + start_time = time.time() # Configure root logger first so that even OmekaContext initialisation # errors are captured at the correct level. @@ -449,12 +457,7 @@ def main(): logger.info("Starting pass 2: record linking") link_items(ctx, pathlist) - # Print a consolidated summary of all per-item errors encountered during - # the run. Exits 0 if no errors; exits 1 if any item failed. The Ruby - # caller (bin/post_omeka) checks the exit code to determine whether the - # run completed cleanly. - ctx.report_errors() - sys.exit(1 if ctx._errors else 0) + finish_run(ctx, args, start_time) if __name__ == "__main__": diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index dad4cb009..160983b96 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -7,6 +7,7 @@ import logging import sys +import time from datetime import date, datetime from pathlib import Path from typing import Dict, List, Optional @@ -490,3 +491,25 @@ def report_errors(self): logger.warning(" %s", err) else: logger.info("Run completed successfully with no errors.") + +def finish_run(ctx, args, start_time): + """ + Report errors, write count to --error-file if provided, print timing, and exit. + + Called at the end of each Omeka entrypoint script's main() function. + Exits 0 on success, 1 if any errors were recorded. + + Parameters: + * ctx - OmekaContext whose _errors list is inspected + * args - argparse.Namespace; checked for optional error_file attribute + * start_time - float from time.time() captured at the top of main() + """ + ctx.report_errors() + if getattr(args, "error_file", None): + with open(args.error_file, "w") as f: + f.write(str(len(ctx._errors))) + elapsed = int(time.time() - start_time) + hours, rem = divmod(elapsed, 3600) + mins, secs = divmod(rem, 60) + print("Script finished in {:02d} hrs {:02d} mins {:02d} secs".format(hours, mins, secs)) + sys.exit(1 if ctx._errors else 0) \ No newline at end of file From f8268d2f010e19973d680ff0ea8dfa5629e678c8 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 16:33:36 -0500 Subject: [PATCH 103/222] move duplicate code into helpers --- bin/post_omeka | 26 +--------------- bin/post_omeka_html | 29 +---------------- lib/datura/helpers.rb | 36 ++++++++++++++++++++++ lib/datura/python/html_and_media_ingest.py | 1 + lib/datura/python/json_to_omeka.py | 1 + 5 files changed, 40 insertions(+), 53 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index c90073f76..f1285fb8b 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -2,8 +2,6 @@ require "datura" require "optparse" -require "shellwords" -require "tempfile" @usage = "Usage: post_omeka -[options]..." @@ -50,26 +48,4 @@ datura_dir = File.join(File.dirname(__FILE__), "..") # path to the gem's config files python_script_path = File.join(datura_dir, "lib", "datura", "python", "json_to_omeka.py") #run posting script into Omeka S -if File.exist?("#{python_script_path}") - error_file = Tempfile.new(["omeka_errors", ".txt"]) - error_file_path = error_file.path - error_file.close - command = ["python3", python_script_path] - if options["environment"] - command.append("-e", Shellwords.escape(options["environment"])) - end - if options["regex"] - command.append("-r", Shellwords.escape(options["regex"])) - end - command.append("--error-file", error_file_path) - system(*command) - omeka_errors = begin - Integer(File.read(error_file_path).strip) - rescue - 0 - end - File.unlink(error_file_path) rescue nil - puts "#{omeka_errors} Omeka posting error(s)" -else - puts("Omeka script not found at #{python_script_path}".red) -end \ No newline at end of file +Datura::Helpers.run_omeka_script(python_script_path, options) \ No newline at end of file diff --git a/bin/post_omeka_html b/bin/post_omeka_html index c7fb89c2a..431f265e6 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -1,8 +1,6 @@ #!/usr/bin/env ruby require "datura" -require "shellwords" -require "tempfile" generate_es = true options = {} @@ -53,29 +51,4 @@ datura_dir = File.join(File.dirname(__FILE__), "..") # path to the gem's config files python_script_path = File.join(datura_dir, "lib", "datura", "python", "html_and_media_ingest.py") #run posting script into Omeka S -if File.exist?("#{python_script_path}") - error_file = Tempfile.new(["omeka_errors", ".txt"]) - error_file_path = error_file.path - error_file.close - command = ["python3", python_script_path] - if options["environment"] - command.append("-e", Shellwords.escape(options["environment"])) - end - if options["regex"] - command.append("-r", Shellwords.escape(options["regex"])) - end - if options["media_skip"] - command.append("-m") - end - command.append("--error-file", error_file_path) - system(*command) - omeka_errors = begin - Integer(File.read(error_file_path).strip) - rescue - 0 - end - File.unlink(error_file_path) rescue nil - puts "#{omeka_errors} Omeka posting error(s)" -else - puts("Omeka script not found at #{python_script_path}".red) -end \ No newline at end of file +Datura::Helpers.run_omeka_script(python_script_path, options) \ No newline at end of file diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index bcc245fff..ace97dab0 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -1,6 +1,8 @@ require 'fileutils' require 'net/http' require 'nokogiri' +require 'shellwords' +require 'tempfile' require 'yaml' module Datura::Helpers @@ -140,4 +142,38 @@ def self.construct_auth_header(options) { "Authorization" => "Basic #{Base64::encode64("#{username}:#{password}")}" } end + def self.run_omeka_script(script_path, options) + ''' + Build and run a Python Omeka posting script, then print an error summary. + + Handles tempfile creation for the error count handoff, common CLI flag + forwarding (-e, -r, -m), and the "N Omeka posting error(s)" output line. + Called by bin/post_omeka and bin/post_omeka_html. + + Parameters: + * script_path - absolute path to the Python script to run + * options - hash of parsed CLI options ("environment", "regex", "media_skip") + ''' + unless File.exist?(script_path) + puts "Omeka script not found at #{script_path}".red + return + end + error_file = Tempfile.new(["omeka_errors", ".txt"]) + error_file_path = error_file.path + error_file.close + command = ["python3", script_path] + command.append("-e", Shellwords.escape(options["environment"])) if options["environment"] + command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] + command.append("-m") if options["media_skip"] + command.append("--error-file", error_file_path) + system(*command) + omeka_errors = begin + Integer(File.read(error_file_path).strip) + rescue + 0 + end + File.unlink(error_file_path) rescue nil + puts "#{omeka_errors} Omeka posting error(s)" + end + end diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 4c6a54790..c0ba006a0 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -29,6 +29,7 @@ import logging import os import sys +import time from pathlib import Path import requests diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 3d2f73b20..148280b26 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -34,6 +34,7 @@ import logging import os import sys +import time import traceback from pathlib import Path From 46a0f35dfe181a21452e7f4a92f89538f6d30a00 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 29 May 2026 17:03:42 -0500 Subject: [PATCH 104/222] use stem rather than full path for regex match --- lib/datura/python/omeka.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 8c9843f06..0f29a1278 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -253,11 +253,11 @@ def filter_items(regex, pathlist): * regex - regex pattern string, compiled with re.compile() * pathlist - iterable of pathlib.Path or string paths to filter - Returns a list containing only the paths whose string representation - matches the pattern. + Returns a list containing only the paths whose stem (identifier, without + extension or directory components) matches the pattern. """ reg = re.compile(regex) - return [p for p in pathlist if reg.search(str(p))] + return [p for p in pathlist if reg.search(p.stem)] def filter_items_by_date(update_time, pathlist): From 5c446462b3fdc909838e86c48171c3be77e0984e Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 1 Jun 2026 09:30:41 -0500 Subject: [PATCH 105/222] add format filter to omeka annex --- bin/post_omeka | 8 ++++++- bin/post_omeka_html | 6 +++++ lib/datura/helpers.rb | 1 + lib/datura/python/html_and_media_ingest.py | 10 +++++++- lib/datura/python/json_to_omeka.py | 10 +++++++- lib/datura/python/omeka.py | 27 ++++++++++++++++++++++ lib/datura/python/omeka_context.py | 4 +++- 7 files changed, 62 insertions(+), 4 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index f1285fb8b..b35fad137 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -20,13 +20,19 @@ optparse = OptionParser.new do |opts| options["environment"] = input end end + + opts.on('-f', '--format [input]', 'only post files of this format (tei, csv, vra, ead, html, pdf, webs)') do |input| + if input && input.length > 0 + options["format"] = input + end + end opts.on('-r', '--regex [input]', 'only generate and post files matching this regex') do |input| if input && input.length > 0 options["regex"] = input end end - + opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| #does nothing, this is a placeholder to catch all arguments not needed now and pass them along to the main script end diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 431f265e6..9b9017dec 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -20,6 +20,12 @@ optparse = OptionParser.new do |opts| options["environment"] = input end end + + opts.on('-f', '--format [input]', 'only post files of this format (tei, csv, vra, ead, html, pdf, webs)') do |input| + if input && input.length > 0 + options["format"] = input + end + end opts.on('-r', '--regex [input]', 'only generate and post files matching this regex') do |input| if input && input.length > 0 diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index ace97dab0..07c364869 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -164,6 +164,7 @@ def self.run_omeka_script(script_path, options) command = ["python3", script_path] command.append("-e", Shellwords.escape(options["environment"])) if options["environment"] command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] + command.append("-f", Shellwords.escape(options["format"])) if options["format"] command.append("-m") if options["media_skip"] command.append("--error-file", error_file_path) system(*command) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index c0ba006a0..02d0a0f87 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -36,7 +36,7 @@ from requests.exceptions import HTTPError import omeka -from omeka import add_media_to_item, filter_items, filter_items_by_date +from omeka import add_media_to_item, filter_items, filter_items_by_date, filter_items_by_format from omeka_context import ( OmekaAPIError, OmekaContext, @@ -74,6 +74,12 @@ def _parse_args(): default="development", help="Target environment: 'development' or 'production' (default: development).", ) + parser.add_argument( + "-f", "--format", + default=None, + dest="format_filter", + help="Only post files of this format (tei, csv, vra, ead, html, pdf, webs).", + ) parser.add_argument( "-r", "--regex", default=None, @@ -456,6 +462,8 @@ def main(): pathlist = list(Path(json_dir).glob("**/*.json")) + if ctx.format_filter: + pathlist = filter_items_by_format(ctx.format_filter, pathlist) if ctx.regex: pathlist = filter_items(ctx.regex, pathlist) if ctx.update_time: diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 148280b26..575fb193c 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -48,7 +48,7 @@ configure_logging, finish_run, ) -from omeka import filter_items, filter_items_by_date, prepare_item_payload_using_template +from omeka import filter_items, filter_items_by_date, filter_items_by_format, prepare_item_payload_using_template # Module-level logger. Records from this module appear as "json_to_omeka" # in log output so they can be filtered independently from other modules. @@ -81,6 +81,12 @@ def _parse_args(): default="development", help="Target environment: 'development' or 'production' (default: development).", ) + parser.add_argument( + "-f", "--format", + default=None, + dest="format_filter", + help="Only post files of this format (tei, csv, vra, ead, html, pdf, webs).", + ) parser.add_argument( "-r", "--regex", default=None, @@ -427,6 +433,8 @@ def main(): json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) pathlist = list(Path(json_dir).glob("**/*.json")) + if ctx.format_filter: + pathlist = filter_items_by_format(ctx.format_filter, pathlist) if ctx.regex: pathlist = filter_items(ctx.regex, pathlist) if ctx.update_time: diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 0f29a1278..1d8b97e76 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -292,3 +292,30 @@ def filter_items_by_date(update_time, pathlist): if source_mtime >= update_time: result.append(p) return result + + + +def filter_items_by_format(format_type, pathlist): + """ + Filter a list of output JSON file paths to those whose source file lives + in the source// directory. + + Mirrors the Ruby DataManager convention: source files for a given format + are stored under source// (e.g. source/csv/, source/tei/). A JSON + output file belongs to that format if and only if a corresponding source + file exists at source//.*. + + Files with no match in source// are excluded. This correctly + drops stale JSON left in the output directory from earlier runs of a + different format. + + Parameters: + * format_type - string, e.g. "tei" or "csv" + * pathlist - iterable of pathlib.Path or string paths to filter + """ + source_dir = Path.cwd() / "source" / format_type + result = [] + for p in pathlist: + if list(source_dir.glob("{}.*".format(Path(p).stem))): + result.append(p) + return result \ No newline at end of file diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 160983b96..3b93aa698 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -231,6 +231,7 @@ def from_args(cls, args): regex=getattr(args, "regex", None), media_skip=getattr(args, "media_skip", False), update_time=parse_update_time(raw_update) if raw_update else None, + format_filter=getattr(args, "format_filter", None), ) @staticmethod @@ -272,7 +273,7 @@ def _load_config(path, env): return contents[env] - def __init__(self, config, env_config, environment, regex, media_skip, update_time=None): + def __init__(self, config, env_config, environment, regex, media_skip, update_time=None, format_filter=None): """ Initialise the context. Prefer OmekaContext.from_args() over calling this constructor directly except in tests. @@ -316,6 +317,7 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti self.regex = regex self.media_skip = media_skip self.update_time = update_time + self.format_filter = format_filter # ---- Config values ------------------------------------------------ self.template_number = config["resource_template"] From 7e14835aaca420b6cb2c49ddb2cafca43b28dd09 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 1 Jun 2026 16:06:46 -0500 Subject: [PATCH 106/222] require item set id --- lib/datura/python/omeka_context.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 3b93aa698..f6a0f98af 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -312,6 +312,18 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti .format(key) ) + # ---- Validate environment-specific item_set --------------------------- + if "item_set" not in env_config: + raise OmekaConfigError( + "Missing 'item_set' for environment {!r} in config/private.yml.\n" + "Add the item set ID for this environment before running. Example:\n\n" + " {}:\n" + " item_set: 123\n\n" + "To find your item set ID, log into the Omeka S admin and navigate " + "to Items > Item Sets." + .format(environment, environment) + ) + # ---- Runtime flags ------------------------------------------------ self.environment = environment self.regex = regex From 51c1067f8db480eed592739c4759fb2667844e14 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 1 Jun 2026 16:13:19 -0500 Subject: [PATCH 107/222] catch KeyboardInterrupt at top level and exit with clear error message --- lib/datura/python/html_and_media_ingest.py | 6 +++++- lib/datura/python/json_to_omeka.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 02d0a0f87..44a0edb84 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -483,4 +483,8 @@ def main(): if __name__ == "__main__": - main() + try: + main() + except KeyboardInterrupt: + print("\nInterrupted. Exiting.") + sys.exit(1) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 575fb193c..b3c9f84d0 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -470,4 +470,8 @@ def main(): if __name__ == "__main__": - main() + try: + main() + except KeyboardInterrupt: + print("\nInterrupted. Exiting.") + sys.exit(1) From d1f1457b40959fb8a2dc859a67d7e861842fe855 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 1 Jun 2026 16:29:11 -0500 Subject: [PATCH 108/222] clarify error msg and exit immediately for HTTPErrors --- lib/datura/python/html_and_media_ingest.py | 14 +++++++++- lib/datura/python/json_to_omeka.py | 5 ++++ lib/datura/python/omeka_context.py | 30 ++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 44a0edb84..c3a923dd3 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -39,6 +39,8 @@ from omeka import add_media_to_item, filter_items, filter_items_by_date, filter_items_by_format from omeka_context import ( OmekaAPIError, + OmekaAuthError, + OmekaConfigError, OmekaContext, OmekaMediaError, OmekaMultipleMatchesError, @@ -153,7 +155,13 @@ def delete_media_items(ctx, matching_item): logger.info("Deleting media item %s", media_id) ctx.client.delete_resource(media_id, "media") except HTTPError as err: - if err.response.status_code == 500: + if err.response.status_code == 401: + raise OmekaAuthError( + "Omeka S returned 401 Unauthorized. " + "Check that key_identity and key_credential in config/private.yml are correct. " + "You may also need to be logged onto the VPN." + ) from err + elif err.response.status_code == 500: # 500 on DELETE is treated as "already gone" by convention. # Log at DEBUG so it does not clutter normal output. logger.debug( @@ -488,3 +496,7 @@ def main(): except KeyboardInterrupt: print("\nInterrupted. Exiting.") sys.exit(1) + except OmekaConfigError as err: + logger.debug("Fatal configuration error:", exc_info=True) + print("ERROR: {}".format(err), file=sys.stderr) + sys.exit(1) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index b3c9f84d0..1b0570ac7 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -42,6 +42,7 @@ import omeka from omeka_context import ( OmekaAPIError, + OmekaConfigError, OmekaContext, OmekaItemNotFoundError, OmekaMultipleMatchesError, @@ -475,3 +476,7 @@ def main(): except KeyboardInterrupt: print("\nInterrupted. Exiting.") sys.exit(1) + except OmekaConfigError as err: + logger.debug("Fatal configuration error:", exc_info=True) + print("ERROR: {}".format(err), file=sys.stderr) + sys.exit(1) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index f6a0f98af..e4222d122 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -73,6 +73,29 @@ class OmekaConfigError(OmekaError): exits immediately rather than attempting to continue. """ +class OmekaAuthError(OmekaConfigError): + """ + Raised when the Omeka S API returns 401 Unauthorized. + + This is always fatal: if credentials are wrong every API call will fail, + so the process exits immediately rather than accumulating per-item failures. + + Common causes: + - key_identity or key_credential is wrong or has been revoked + - The Omeka S instance URL points to the wrong server + """ + + +def _is_unauthorized(err): + """Return True if err is an HTTP 401 response error from the requests library.""" + try: + from requests.exceptions import HTTPError + return isinstance(err, HTTPError) and getattr( + getattr(err, "response", None), "status_code", None + ) == 401 + except ImportError: + return False + class OmekaAPIError(OmekaError): """ @@ -485,6 +508,13 @@ def record_error(self, err): Parameters: * err - an OmekaError (or subclass) instance describing the failure """ + cause = getattr(err, "cause", None) + if _is_unauthorized(cause): + raise OmekaAuthError( + "Omeka S returned 401 Unauthorized. " + "Check that key_identity and key_credential in config/private.yml are correct. " + "You may also need to be logged onto the VPN." + ) from cause logger.error(str(err)) self._errors.append(err) From fd34bdffcd3f44b5b638b6403b69d9c8281d8ff9 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 2 Jun 2026 11:14:34 -0500 Subject: [PATCH 109/222] fix typo --- docs/2_customization/omeka_overrides.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/2_customization/omeka_overrides.md b/docs/2_customization/omeka_overrides.md index 202cd1b2e..3d6226a71 100644 --- a/docs/2_customization/omeka_overrides.md +++ b/docs/2_customization/omeka_overrides.md @@ -6,7 +6,7 @@ Each Omeka field is updated by the method in [api_fields.py](../../../lib/datura ### Overriding fields -To override the field definitions, copy the file [omeka_overrides_example.py](../../../lib/datura/python/omeka_overrides_example.py) to [omeka_overrides.rb](../../../lib/datura/python/omeka_overrides.py) in the `scripts/overrides` file of the project directory. Then override each method as needed, using the existing definitions in `field_definitions.py` as examples. For an example, see https://github.com/CDRH/data_stories_humanity/blob/omeka_s_ingest/scripts/python/omeka_overrides.py (not currently field). +To override the field definitions, copy the file [omeka_overrides_example.py](../../../lib/datura/python/omeka_overrides_example.py) to [omeka_overrides.py](../../../lib/datura/python/omeka_overrides.py) in the `scripts/overrides` file of the project directory. Then override each method as needed, using the existing definitions in `field_definitions.py` as examples. For an example, see https://github.com/CDRH/data_stories_humanity/blob/omeka_s_ingest/scripts/python/omeka_overrides.py (not currently field). Each overriden method needs to take the arguments `self` (a Python placeholder for a class instance) and `json` (representing the generated JSON) and to match the methods defined on `field_definitions.py`. (The same goes for adding new methods to `field_definitions.py`.) For instance: From eba0682eac9aeb22a42d259f5e80a6da1bfaa4b9 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 2 Jun 2026 12:35:20 -0500 Subject: [PATCH 110/222] adjust error handling: add log file and color to terminal output --- lib/datura/python/omeka_context.py | 74 ++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index e4222d122..b3bd6eb68 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -24,25 +24,46 @@ # Logging setup # --------------------------------------------------------------------------- +# --- Colored console handler --- +CYAN = "\033[36m" +GREEN = "\033[32m" +RED = "\033[31m" +RED2 = "\033[1;31m" +YELLOW = "\033[33m" +RESET = "\033[0m" + +class ColoredConsoleHandler(logging.StreamHandler): + COLORS = { + logging.DEBUG: CYAN, + logging.INFO: GREEN, + logging.WARNING: YELLOW, + logging.ERROR: RED, + logging.CRITICAL: RED2, + } + RESET = RESET + def emit(self, record): + color = self.COLORS.get(record.levelno, self.RESET) + record.msg = f"{color}{record.msg}{self.RESET}" + super().emit(record) + def configure_logging(level="INFO"): - """ - Configure the root logger for the pipeline. + numeric_level = getattr(logging, level.upper(), logging.INFO) - Should be called once at the very start of each entrypoint script before - any other work begins. Subsequent calls are safe but have no additional - effect — Python's logging.basicConfig() is a no-op if handlers are already - attached to the root logger. + # File handler — verbose + file_handler = logging.FileHandler("logs/python.log") + file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")) + file_handler.setLevel(logging.DEBUG) # always save everything - Parameters: - * level - logging level string: "DEBUG", "INFO", "WARNING", or "ERROR". - Defaults to "INFO". Use "DEBUG" to trace individual API calls - and property ID cache hits/misses. - """ - logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", - level=getattr(logging, level.upper(), logging.INFO), - ) + # Console handler — colored, respects the requested level + console_handler = ColoredConsoleHandler() + console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(name)s: %(message)s")) + console_handler.setLevel(numeric_level) + # Root logger + root = logging.getLogger() + root.setLevel(logging.DEBUG) # let handlers decide what to filter + root.addHandler(file_handler) + root.addHandler(console_handler) # --------------------------------------------------------------------------- # Exception hierarchy @@ -174,10 +195,11 @@ def parse_update_time(s): return datetime.strptime(s, fmt) except ValueError: continue - raise OmekaConfigError( + raise OmekaConfigError(RED + "Invalid --update value {!r}. " "Expected 'today', a date (2015-01-01), or date-time (2015-01-01T18:24)." .format(s) + + RESET ) @@ -276,22 +298,25 @@ def _load_config(path, env): with open(path) as f: contents = yaml.safe_load(f) except FileNotFoundError: - raise OmekaConfigError( + raise OmekaConfigError(RED + "Config file not found: {}. " "Ensure config/private.yml exists in the collection directory " "and that you are running the script from the collection root." .format(path) + + RESET ) except yaml.YAMLError as exc: - raise OmekaConfigError( + raise OmekaConfigError(RED + "Could not parse YAML in {}: {}".format(path, exc) + + RESET ) if env not in contents: - raise OmekaConfigError( + raise OmekaConfigError(RED + "Environment section {!r} not found in {}. " "Available sections: {}" .format(env, path, list(contents.keys())) + + RESET ) return contents[env] @@ -329,15 +354,16 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti ] for key in required_keys: if key not in config: - raise OmekaConfigError( + raise OmekaConfigError(RED + "Missing required config key {!r}. " "Check the 'default' section of config/private.yml." .format(key) + + RESET ) # ---- Validate environment-specific item_set --------------------------- if "item_set" not in env_config: - raise OmekaConfigError( + raise OmekaConfigError(RED + "Missing 'item_set' for environment {!r} in config/private.yml.\n" "Add the item set ID for this environment before running. Example:\n\n" " {}:\n" @@ -345,6 +371,7 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti "To find your item set ID, log into the Omeka S admin and navigate " "to Items > Item Sets." .format(environment, environment) + + RESET ) # ---- Runtime flags ------------------------------------------------ @@ -510,10 +537,11 @@ def record_error(self, err): """ cause = getattr(err, "cause", None) if _is_unauthorized(cause): - raise OmekaAuthError( + raise OmekaAuthError(RED + "Omeka S returned 401 Unauthorized. " "Check that key_identity and key_credential in config/private.yml are correct. " "You may also need to be logged onto the VPN." + + RESET ) from cause logger.error(str(err)) self._errors.append(err) @@ -555,5 +583,5 @@ def finish_run(ctx, args, start_time): elapsed = int(time.time() - start_time) hours, rem = divmod(elapsed, 3600) mins, secs = divmod(rem, 60) - print("Script finished in {:02d} hrs {:02d} mins {:02d} secs".format(hours, mins, secs)) + print(f"{CYAN}Script finished in {hours:02d} hrs {mins:02d} mins {secs:02d} secs{RESET}") sys.exit(1 if ctx._errors else 0) \ No newline at end of file From d8b4b65565d718cc46d3d6474d45acc06f47af7b Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 2 Jun 2026 14:11:01 -0500 Subject: [PATCH 111/222] adjust path for overrides file --- lib/datura/python/omeka_overrides_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/omeka_overrides_example.py b/lib/datura/python/omeka_overrides_example.py index 1c1a93262..8020c2ca2 100644 --- a/lib/datura/python/omeka_overrides_example.py +++ b/lib/datura/python/omeka_overrides_example.py @@ -1,4 +1,4 @@ -#copy this file to omeka_overrides.py in your scripts/overrides file. Edit the return values as needed +#copy this file to omeka_overrides.py in your scripts/python directory. Edit the return values as needed from field_definitions import FieldDefinitions From 4fe4628121f0221a25cf416a4d2451f65310ca0a Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 2 Jun 2026 14:40:45 -0500 Subject: [PATCH 112/222] remove omeka_s_tools module function overrides (will be updated in fork) --- lib/datura/python/api_fields.py | 8 +- lib/datura/python/html_and_media_ingest.py | 8 +- lib/datura/python/omeka.py | 155 --------------------- 3 files changed, 5 insertions(+), 166 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 24bfc22da..3ce696566 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -327,10 +327,7 @@ def add_formatted_value(ctx, item, key, value, datatype, label=""): "value": value, "type": datatype, } - # Use the custom prepare_property_value from omeka.py, which supports the - # label parameter for URI types. For resource:item links, use - # ctx.client.prepare_property_value() instead (see link_item_record). - formatted = omeka.prepare_property_value(prop_value, prop_id, label) + formatted = ctx.client.prepare_property_value(prop_value, prop_id, label) if key in item and type(item[key]) == list: item[key].append(formatted) @@ -518,9 +515,6 @@ def link_item_record(ctx, item, key, values, item_set=False, filter_property="dc "type": resource_type, "value": omeka_id, } - # Use the library's prepare_property_value (via ctx.client) for - # resource links, not the custom omeka.py version — the library - # version correctly handles the value_resource_id field. formatted = ctx.client.prepare_property_value(prop_value, prop_id) if item_set: diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index c3a923dd3..b006390b2 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -36,7 +36,7 @@ from requests.exceptions import HTTPError import omeka -from omeka import add_media_to_item, filter_items, filter_items_by_date, filter_items_by_format +from omeka import filter_items, filter_items_by_date, filter_items_by_format from omeka_context import ( OmekaAPIError, OmekaAuthError, @@ -261,7 +261,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): "o:is_public": ctx.is_public, "data": { "upload": str(thumbnail_local), - "dcterms:title": omeka.prepare_property_value( + "dcterms:title": ctx.client.prepare_property_value( json_item.get("title", ""), ctx.get_property_id("dcterms:title"), ), @@ -269,7 +269,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): "o:ingester": "upload", } logger.info("Posting thumbnail for %r", identifier) - add_media_to_item(ctx, matching_item["o:id"], thumbnail_local, payload=media_payload) + ctx.client.add_media_to_item(matching_item["o:id"], thumbnail_local, payload=media_payload) except FileNotFoundError: # The download step wrote the file, but something removed it between # download and upload. Unlikely in practice but handled explicitly @@ -342,7 +342,7 @@ def ingest_html(ctx, json_item, matching_item, html_dir): try: logger.info("Posting HTML for %r", identifier) - add_media_to_item(ctx, matching_item["o:id"], file_path, payload=media_payload) + ctx.client.add_media_to_item(matching_item["o:id"], file_path, payload=media_payload) except Exception as err: ctx.record_error( OmekaMediaError( diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 1d8b97e76..73033625c 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -7,101 +7,12 @@ from datetime import datetime from pathlib import Path -import json import logging import os import re logger = logging.getLogger(__name__) -def add_media_to_item(ctx, item_id, media_file, payload=None, template_id=None, class_id=None): - """ - Upload a media file and associate it with an existing Omeka S item. - - This is a modified version of the omeka-s-tools library method. The key - difference is that the ingester type ("upload", "html", etc.) is read from - payload["o:ingester"] rather than always defaulting to "upload". This allows - the same function to handle both binary file uploads and the HTML ingester, - which reads content from payload["data"]["html"] instead of a file. - - Parameters: - * ctx - OmekaContext providing the authenticated API client - * item_id - numeric Omeka ID of the item this media should attach to - * media_file - path to the media file as a string or pathlib.Path. - For the HTML ingester, this is the path to the .html file, - although the Omeka API reads content from payload["data"]["html"] - rather than the uploaded bytes. - * payload - dict of metadata for the media object. Must contain - "o:ingester" (e.g. "upload" or "html") and any additional - metadata fields. Defaults to an empty dict. - * template_id - optional numeric Omeka resource template ID to attach - to the media object (rarely needed for media). - * class_id - optional numeric Omeka resource class ID. If template_id - is given and class_id is not, the class is inferred from - the template automatically. - - Returns the Omeka JSON-LD representation of the newly created media object. - """ - if payload is None: - payload = {} - - files = {} - - # Legacy dict-style call: {"path": ..., "title": ...} - # Preserved for backwards compatibility with any callers using the older - # interface from the omeka-s-tools library. - if isinstance(media_file, dict): - path = media_file['path'] - payload = media_file['title'] - - # Normalise the path to a pathlib.Path regardless of input type. - path = Path(media_file) - - # If a bare string title was passed as the payload, wrap it in the - # standard item payload format expected by the API. - if isinstance(payload, str): - payload = ctx.client.prepare_item_payload({'dcterms:title': [payload]}) - - # Attach resource template metadata if requested. - if template_id: - payload['o:resource_template'] = ctx.client.format_resource_id( - template_id, 'resource_templates' - ) - if not class_id: - # Infer the resource class from the template when not supplied. - template = ctx.client.get_resource_by_id(template_id, 'resource_templates') - class_id = template['o:resource_class']['o:id'] - if class_id: - payload['o:resource_class'] = ctx.client.format_resource_id( - class_id, 'resource_classes' - ) - - # Use the ingester declared in the payload, falling back to "upload". - # Using .get() guards against a missing key - ingester = payload.get("o:ingester") or "upload" - - # Core fields required by Omeka S for any media POST. - file_data = { - 'o:ingester': ingester, - 'file_index': '0', # index into the files[] multipart array - 'o:source': path.name, # original filename, shown in Omeka admin - 'o:item': {'o:id': item_id}, - } - payload.update(file_data) - - # Read the raw file bytes and attach them as file[0] in the multipart body. - # For the HTML ingester, Omeka reads content from payload["data"]["html"] - # and ignores the file bytes, but including them does not cause errors. - files['file[0]'] = path.read_bytes() - files['data'] = (None, json.dumps(payload), 'application/json') - - response = ctx.client.s.post( - '{}/media'.format(ctx.client.api_url), - files=files, - params=ctx.client.credentials, - ) - return ctx.client.process_response(response) - def prepare_item_payload_using_template(ctx, terms, template_id): """ @@ -176,72 +87,6 @@ def prepare_item_payload_using_template(ctx, terms, template_id): return payload - -def prepare_property_value(value, property_id, label=""): - """ - Format a single property value in the structure expected by Omeka S. - - This is a custom version of the omeka-s-tools library method, extended to - support an optional text label for URI-type values. It is used in - api_fields.add_formatted_value() for all standard property formatting. - - Parameters: - * value - a string, int, float, or dict. Non-dict values are - automatically wrapped: {"value": }. Dicts may - include a "type" key; if absent, "literal" is used. - * property_id - numeric Omeka property ID for this term - * label - display label for URI values. If omitted, the last path - segment of the URI is used as the label. - - Returns a dict formatted for inclusion in an Omeka S item payload. - - NOTE: The "resource:item" branch contains a reference to `self.api_url` - which is a pre-existing copy-paste bug from the library source (this is a - standalone function, not a method, so `self` is undefined). This branch - is not reached by any current pipeline caller — all values are "literal" - or "uri" — so the bug has been left in place with this comment rather than - silently changing potentially-load-bearing code during a refactor. - If you need resource:item linking, use ctx.client.prepare_property_value() - (the library version) instead. - """ - # Wrap bare scalars so the rest of the function can assume a dict. - if not isinstance(value, dict): - value = {'value': value} - - # Default to "literal" when no explicit type is provided. - try: - data_type = value['type'] - except KeyError: - data_type = 'literal' - - property_value = { - 'property_id': property_id, - 'type': data_type, - } - - if data_type == 'resource:item': - # This branch is intentionally not implemented in this standalone function. - # Use ctx.client.prepare_property_value() for resource:item values instead — - # the library version has access to the API URL via the client instance. - raise NotImplementedError( - "resource:item values must use ctx.client.prepare_property_value(); " - "see link_item_record() in api_fields.py" - ) - elif data_type == 'uri': - property_value['@id'] = value['value'] - # Fall back to the last URI segment when no explicit label is given. - if label == "": - property_value["o:label"] = value["value"].split("/")[-1] - else: - property_value["o:label"] = label - else: - # "literal", "numeric:timestamp", and any other types store the - # value under the "@value" key. - property_value['@value'] = value['value'] - - return property_value - - def filter_items(regex, pathlist): """ Filter a list of file paths to those matching a regex pattern. From faca42baa151f7b09abccc97523b0555b562e6d6 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 3 Jun 2026 15:33:04 -0500 Subject: [PATCH 113/222] merge configs so env config overrides default --- lib/datura/python/omeka_context.py | 40 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index b3bd6eb68..ffb95fc3f 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -266,9 +266,13 @@ def from_args(cls, args): ) env_config = {} + # Merge so that environment-specific values override defaults, giving + # collections the ability to override any default key (e.g. + # resource_template, omeka_data_base) on a per-environment basis. + env_config = {**default_config, **env_config} + raw_update = getattr(args, "update_time", None) return cls( - config=default_config, env_config=env_config, environment=args.environment, # getattr with a default handles entrypoints that don't define @@ -321,17 +325,17 @@ def _load_config(path, env): return contents[env] - def __init__(self, config, env_config, environment, regex, media_skip, update_time=None, format_filter=None): + def __init__(self, env_config, environment, regex, media_skip, update_time=None, format_filter=None): """ Initialise the context. Prefer OmekaContext.from_args() over calling this constructor directly except in tests. Parameters: - * config - dict from the "default" section of private.yml; - must contain omeka_server, key_identity, key_credential, - resource_template, and omeka_data_base - * env_config - dict from the environment-specific section; used to - look up item_set + * env_config - merged dict: the "default" section of private.yml + overlaid with the environment-specific section so that + per-environment values take precedence over defaults. + Must contain omeka_server, key_identity, key_credential, + resource_template, omeka_data_base, and item_set. * environment - "development" or "production" * regex - optional regex string to filter input file paths; None means process all files in the output directory @@ -353,11 +357,11 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti "omeka_data_base", ] for key in required_keys: - if key not in config: + if key not in env_config: raise OmekaConfigError(RED + "Missing required config key {!r}. " - "Check the 'default' section of config/private.yml." - .format(key) + "Check the 'default' or {!r} section of config/private.yml." + .format(key, environment) + RESET ) @@ -382,22 +386,22 @@ def __init__(self, config, env_config, environment, regex, media_skip, update_ti self.format_filter = format_filter # ---- Config values ------------------------------------------------ - self.template_number = config["resource_template"] - self.omeka_data_base = config["omeka_data_base"] + self.template_number = env_config["resource_template"] + self.omeka_data_base = env_config["omeka_data_base"] # iiif_server is optional — not all collections ingest thumbnails. - self.iiif_server = config.get("iiif_server", "") + self.iiif_server = env_config.get("iiif_server", "") # iiif_collection is optional — not all collections have different iiif collection names. - self.iiif_collection = config.get("iiif_collection", "") + self.iiif_collection = env_config.get("iiif_collection", "") - # Keep the environment-specific dict for the item_set_id property. + # Keep the merged config dict for the item_set_id property. self._env_config = env_config # ---- Credentials (stored for reset_client) ------------------------ # Stored privately so that credential strings are not accidentally # printed, logged, or serialised through the public interface. - self._api_url = config["omeka_server"] - self._key_identity = config["key_identity"] - self._key_credential = config["key_credential"] + self._api_url = env_config["omeka_server"] + self._key_identity = env_config["key_identity"] + self._key_credential = env_config["key_credential"] # ---- API client --------------------------------------------------- # Single authenticated client used for all API operations. From 47b95d1ad49c8ccce39bd10e505c3675b3cb9e34 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 3 Jun 2026 15:59:47 -0500 Subject: [PATCH 114/222] add Python setuptools version for omeka_s_tools installation --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7172dfdb5..4e2929a94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ platformdirs==4.4.0 python-dotenv==1.1.1 requests==2.32.5 requests-cache==1.2.1 +setuptools==82.0.1 typing_extensions==4.15.0 url-normalize==2.2.1 urllib3==2.5.0 From c94ccf75489af485673248c15f0984c12e0e4c85 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 4 Jun 2026 16:49:13 -0500 Subject: [PATCH 115/222] shift JSON info message to relative path --- lib/datura/python/json_to_omeka.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 1b0570ac7..7d90fdeef 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -444,7 +444,7 @@ def main(): logger.info( "Found %d JSON file(s) in %s (environment=%r)", len(pathlist), - json_dir, + "output/{}/es".format(ctx.environment), ctx.environment, ) From 359633d54cc0b3742e55692e187eacd92f72ee66 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 4 Jun 2026 16:49:42 -0500 Subject: [PATCH 116/222] add info message to alert user that overrides file is in use --- lib/datura/python/field_definitions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 19604a08c..c6b7c5aad 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -1,7 +1,10 @@ +import logging import sys import os from datetime import datetime +logger = logging.getLogger(__name__) + class FieldDefinitions: """ Default field extraction patterns for the Omeka S ingestion pipeline. @@ -270,6 +273,7 @@ def get_fields(omeka_data_base=""): # CustomFields inherits __init__ from FieldDefinitions, so # omeka_data_base is passed through automatically. Override __init__ # in CustomFields only if you need additional constructor logic. + logger.info("Omeka overrides found at %s; custom field mappings will be applied.", "scripts/python/omeka_overrides.py") return CustomFields(omeka_data_base=omeka_data_base) except ImportError: # No collection-specific overrides found; use the defaults. From 6794f65d5c9def56b444ffa49bf8db7cf7ad03fc Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 4 Jun 2026 17:01:45 -0500 Subject: [PATCH 117/222] use RotatingFileHandler to manage size of log file --- lib/datura/python/omeka_context.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index ffb95fc3f..47e6778ce 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -6,6 +6,7 @@ """ import logging +from logging.handlers import RotatingFileHandler import sys import time from datetime import date, datetime @@ -50,7 +51,9 @@ def configure_logging(level="INFO"): numeric_level = getattr(logging, level.upper(), logging.INFO) # File handler — verbose - file_handler = logging.FileHandler("logs/python.log") + file_handler = RotatingFileHandler( + "logs/python.log", maxBytes=5 * 1024 * 1024, backupCount=3 + ) file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")) file_handler.setLevel(logging.DEBUG) # always save everything From e1be997b34088d379fd9470e024463199ebed1a5 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 09:06:22 -0500 Subject: [PATCH 118/222] make logs dir if it does not exist --- lib/datura/python/omeka_context.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 47e6778ce..185d39b08 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -7,6 +7,7 @@ import logging from logging.handlers import RotatingFileHandler +import os import sys import time from datetime import date, datetime @@ -49,6 +50,8 @@ def emit(self, record): def configure_logging(level="INFO"): numeric_level = getattr(logging, level.upper(), logging.INFO) + + os.makedirs("logs", exist_ok=True) # File handler — verbose file_handler = RotatingFileHandler( From 33a5eb3e182cc60a122736d448476335682158a4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 09:07:01 -0500 Subject: [PATCH 119/222] shift info message about overrides to warning --- lib/datura/python/field_definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index c6b7c5aad..9fda948e9 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -273,7 +273,7 @@ def get_fields(omeka_data_base=""): # CustomFields inherits __init__ from FieldDefinitions, so # omeka_data_base is passed through automatically. Override __init__ # in CustomFields only if you need additional constructor logic. - logger.info("Omeka overrides found at %s; custom field mappings will be applied.", "scripts/python/omeka_overrides.py") + logger.warning("Omeka overrides found at %s; custom field mappings will be applied.", "scripts/python/omeka_overrides.py") return CustomFields(omeka_data_base=omeka_data_base) except ImportError: # No collection-specific overrides found; use the defaults. From 5057c9658003ea6cc1b4732dfdc86f2ec7b31f97 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 09:49:25 -0500 Subject: [PATCH 120/222] move overrides get_fields call to context so every item does not call it --- lib/datura/python/api_fields.py | 10 ++++------ lib/datura/python/json_to_omeka.py | 14 ++++---------- lib/datura/python/omeka_context.py | 9 +++++++++ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 3ce696566..2caaddce0 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -21,7 +21,6 @@ import omeka from datetime import datetime -from field_definitions import get_fields # Module-level logger so that log records from this module are identifiable # by name in the output stream. @@ -48,11 +47,10 @@ def build_item_dict(ctx, json_item, existing_item): unexpected way. """ try: - # Load the collection-specific field definitions, falling back to the - # defaults if no omeka_overrides.py exists in scripts/python/. - # Pass omeka_data_base so that uriData() can construct media URIs - # without needing a global. - fields = get_fields(omeka_data_base=ctx.omeka_data_base) + # Load the collection-specific field definitions once during OmekaContext + # initialization (collection-specific CustomFields subclass if omeka_overrides.py + # is present in scripts/python, otherwise the defaultFieldDefinitions). + fields = ctx.fields # Start from the existing Omeka item dict when updating, or an empty # dict when creating. update_item_value() clears each key before diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 7d90fdeef..d373b8343 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -406,12 +406,10 @@ def main(): error message if the config is missing or malformed (OmekaConfigError). 4. Discover JSON files under output//es/. 5. Apply regex filter if -r was passed. - 6. Add scripts/python to sys.path so that collection-specific - omeka_overrides.py can be imported by field_definitions.get_fields(). - 7. Run pass 1 (post_items). - 8. Reset the API client between passes for a clean connection. - 9. Run pass 2 (link_items). - 10. Print run summary; exit 1 if any per-item errors were recorded, + 6. Run pass 1 (post_items). + 7. Reset the API client between passes for a clean connection. + 8. Run pass 2 (link_items). + 9. Print run summary; exit 1 if any per-item errors were recorded, 0 if all items succeeded. """ args = _parse_args() @@ -448,10 +446,6 @@ def main(): ctx.environment, ) - # Make the collection's scripts/python directory importable so that - # field_definitions.get_fields() can find omeka_overrides.py if present. - sys.path.append(os.path.join(os.getcwd(), "scripts/python")) - # --- Pass 1: create / update items --- logger.info("Starting pass 1: item posting") post_items(ctx, pathlist) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 185d39b08..f12956afc 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -9,6 +9,8 @@ from logging.handlers import RotatingFileHandler import os import sys + +from field_definitions import get_fields import time from datetime import date, datetime from pathlib import Path @@ -433,6 +435,13 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, # run. report_errors() logs a consolidated summary at the end. self._errors = [] # type: List[OmekaError] + # ---- Field definitions -------------------------------------------- + # Load collection-specific field mappings once here. get_fields() returns + # a CustomFields subclass if scripts/python/omeka_overrides.py is present; + # otherwise the default FieldDefinitions instance. + self.fields = get_fields(omeka_data_base=self.omeka_data_base) + + # ----------------------------------------------------------------------- # Properties # ----------------------------------------------------------------------- From 96b516850e3a801a84f45680178ccb619512349d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 10:20:49 -0500 Subject: [PATCH 121/222] shift type checks to isinstance, per linter --- lib/datura/python/api_fields.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 2caaddce0..7d7900fa5 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -327,7 +327,7 @@ def add_formatted_value(ctx, item, key, value, datatype, label=""): } formatted = ctx.client.prepare_property_value(prop_value, prop_id, label) - if key in item and type(item[key]) == list: + if key in item and isinstance(item[key],list): item[key].append(formatted) else: item[key] = [formatted] @@ -353,7 +353,7 @@ def get_matching_ids_from_markdown(row, field): markdown_values = sorted(get_json_value(row, field)) ids = [] if markdown_values: - if type(markdown_values) == str: + if isinstance(markdown_values,str): match = re.search(r"\]\((.*)\)", markdown_values) if match: ids.append(match.group(1)) @@ -392,7 +392,7 @@ def get_matching_names_from_markdown(row, field): markdown_values = get_json_value(row, field) names = [] if markdown_values: - if type(markdown_values) == str: + if isinstance(markdown_values,str): name_match = re.search(r"\[(.*?)\]", markdown_values) id_match = re.search(r"\]\((.*)\)", markdown_values) # Only collect the name if there is no associated identifier. From 085a34a8ea89615c137c074327b50740ee70d617 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 12:22:42 -0500 Subject: [PATCH 122/222] update pkgs, remove unused pkgs --- datura.gemspec | 6 +++--- lib/datura/file_types/file_custom.rb | 2 -- lib/datura/file_types/file_ead.rb | 1 - lib/datura/file_types/file_html.rb | 1 - lib/datura/file_types/file_tei.rb | 1 - lib/datura/file_types/file_webs.rb | 2 -- requirements.txt | 1 - 7 files changed, 3 insertions(+), 11 deletions(-) diff --git a/datura.gemspec b/datura.gemspec index 492818001..fec7219e2 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -57,12 +57,12 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = "~> 3.1" - spec.add_runtime_dependency "colorize", "~> 1.0" + spec.add_runtime_dependency "colorize", "~> 1.1" spec.add_runtime_dependency "nokogiri", "~> 1.18" spec.add_runtime_dependency "rest-client", "~> 2.1" - spec.add_runtime_dependency "pdf-reader", "~> 2.12" + spec.add_runtime_dependency "pdf-reader", "~> 2.15" spec.add_development_dependency "byebug", "~> 11.0" - spec.add_development_dependency "bundler", ">= 2.0", "< 5.0" + spec.add_development_dependency "bundler", ">= 2.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" end diff --git a/lib/datura/file_types/file_custom.rb b/lib/datura/file_types/file_custom.rb index 28725d02b..3fba3cfac 100644 --- a/lib/datura/file_types/file_custom.rb +++ b/lib/datura/file_types/file_custom.rb @@ -1,8 +1,6 @@ require_relative "../helpers.rb" require_relative "../file_type.rb" -require "rest-client" - class FileCustom < FileType attr_reader :es_req, :format diff --git a/lib/datura/file_types/file_ead.rb b/lib/datura/file_types/file_ead.rb index a809ab9b4..7c0fe5c81 100644 --- a/lib/datura/file_types/file_ead.rb +++ b/lib/datura/file_types/file_ead.rb @@ -1,7 +1,6 @@ require_relative "../helpers.rb" require_relative "../file_type.rb" require_relative "../solr_poster.rb" -require "rest-client" class FileEad < FileType # TODO we could include the tei_to_es and other modules directly here diff --git a/lib/datura/file_types/file_html.rb b/lib/datura/file_types/file_html.rb index 4835ba40c..cd820c8f5 100644 --- a/lib/datura/file_types/file_html.rb +++ b/lib/datura/file_types/file_html.rb @@ -1,7 +1,6 @@ require_relative "../helpers.rb" require_relative "../file_type.rb" require_relative "../solr_poster.rb" -require "rest-client" class FileHtml < FileType attr_reader :es_req diff --git a/lib/datura/file_types/file_tei.rb b/lib/datura/file_types/file_tei.rb index d756450f2..463b1ba48 100644 --- a/lib/datura/file_types/file_tei.rb +++ b/lib/datura/file_types/file_tei.rb @@ -1,7 +1,6 @@ require_relative "../helpers.rb" require_relative "../file_type.rb" require_relative "../solr_poster.rb" -require "rest-client" class FileTei < FileType # TODO we could include the tei_to_es and other modules directly here diff --git a/lib/datura/file_types/file_webs.rb b/lib/datura/file_types/file_webs.rb index 62c8dc5d9..28bfcee49 100644 --- a/lib/datura/file_types/file_webs.rb +++ b/lib/datura/file_types/file_webs.rb @@ -1,8 +1,6 @@ require_relative "../helpers.rb" require_relative "../file_type.rb" -require "rest-client" - class FileWebs < FileType attr_reader :es_req diff --git a/requirements.txt b/requirements.txt index 1add69219..ecac30ce6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ Markdown==3.8.2 omeka_s_tools @ git+https://github.com/CDRH/omeka_s_tools.git@will_changes packaging==25.0 platformdirs==4.4.0 -python-dotenv==1.1.1 PyYAML==6.0.2 requests==2.32.5 requests-cache==1.2.1 From b27592e43257ad6cfeae17b85dac7b7d65505300 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 12:39:46 -0500 Subject: [PATCH 123/222] add python/omeka requirements and notes to setup --- bin/setup | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/bin/setup b/bin/setup index 61d5d80a3..383c9b901 100755 --- a/bin/setup +++ b/bin/setup @@ -25,12 +25,23 @@ File.open(File.join(coll, "config", "private.yml"), "w") do |file| default: # number of files processed at the same time, increase for more powerful computers threads: 5 + + # Omeka S posting (required only if using post_omeka / post_omeka_html) + # omeka_server: servername.unl.edu/path/to/api + # key_identity: your_api_key_identity + # key_credential: your_api_key_credential + # resource_template: 1 + # omeka_data_base: path/to/tei + # iiif_server: servername.unl.edu # optional, only needed for media ingest + # iiif_collection: collection_name # optional, only needed for media ingest development: es_path: https://edit_private_config/elastic es_index: edit_private_config + # item_set: 123 production: es_path: https://edit_private_config/elastic es_index: edit_private_config + # item_set: 456 TEXT file.write(text) @@ -81,11 +92,16 @@ end # SCRIPTS -puts "-- Place XSLT, Ruby, and Python overrides in scripts/overrides" +puts "-- Place XSLT and Ruby overrides in scripts/overrides" FileUtils.mkdir_p(File.join(coll, "scripts", "overrides")) FileUtils.touch(File.join(coll, "scripts", "overrides", ".keep")) -FileUtils.cp(File.join(datura, "lib", "datura", "python", "omeka_overrides_example.py"), File.join(coll, "scripts", "overrides", "omeka_overrides_example.py")) +puts "-- Place Python overrides in scripts/python" +FileUtils.mkdir_p(File.join(coll, "scripts", "python")) +FileUtils.cp(File.join(datura, "lib", "datura", "python", "omeka_overrides_example.py"), File.join(coll, "scripts", "python", "omeka_overrides_example.py")) + +puts "-- Copying requirements.txt for Python/Omeka pipeline".green +FileUtils.cp(File.join(datura, "requirements.txt"), File.join(coll, "requirements.txt")) # SOURCE @@ -104,6 +120,7 @@ FileUtils.touch(File.join(src, "drafts", "tei", ".keep")) File.open(File.join(coll, ".gitignore"), "w") do |file| text = < Date: Fri, 5 Jun 2026 14:25:14 -0500 Subject: [PATCH 124/222] fix has_relation field to handle null values --- lib/datura/python/field_definitions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 9fda948e9..ec5398300 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -100,7 +100,8 @@ def dcterms_format(self, json): return json.get("format", None) def relation(self, json): - relation_ids = [relation['id'] for relation in json.get("has_relation") or [] if 'id' in relation] + relations = json.get("has_relation") or {} + relation_ids = [relations['id']] if relations.get('id') is not None else [] return relation_ids def publisher(self, json): From e8757b53afd80bad6a5bca709357c51ef288d7f4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 15:32:01 -0500 Subject: [PATCH 125/222] update omeka setup docs --- docs/1_setup/omeka_setup.md | 46 +++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/1_setup/omeka_setup.md b/docs/1_setup/omeka_setup.md index 23857fb76..bb06dafd6 100644 --- a/docs/1_setup/omeka_setup.md +++ b/docs/1_setup/omeka_setup.md @@ -1,18 +1,35 @@ ## Set up for Omeka S posting -### Setting up data repo for Omeka +### Step 1: Setting up a data repository for Omeka -In your Gemfile, make sure that Datura is on the right branch for Omeka posting. Currently the line should read `gem "datura", git: "https://github.com/CDRH/datura", branch: "omeka_posting_generalized"` (soon it might be incorporated into a formal release). Change `.ruby-gemset` to `datura-omeka` or something similar if this is not the version of Datura you usually use. Run `cd .` and then `bundle install`. Then run `setup`. Copy `omeka_overrides_examples.py` to `omeka_overrides.py` and make desired changes. Alternatively, if you don't want to set up a repo from scratch, clone the `https://github.com/CDRH/datura` repo, switch to the `https://github.com/CDRH/datura` branch, and copy the `lib/datura/python/omeka_overrides_example.py` to `[collection-directory]/scripts/omeka_overrides.py`. +#### If you would like to create a new repository -### Enabling a virtual environment +Follow [the steps](https://github.com/CDRH/datura/blob/dev/docs/1_setup/collection_setup.md#step-1--create-a-new-collection-directory) in the `collection_setup` documentation for Datura, specifying any recent release of Datura in your Gemfile. If you know out of the gate that you plan to create overrides for any Omeka fields, copy `omeka_overrides_examples.py` (in the `/scripts/python` directory) to `omeka_overrides.py`. -In your collection repo, first exit any virtual environemt if one currently enabled (this may be indicated by `(.venv)` or similar text before your command prompt) with `deactivate`. If you have not previously created a virtual environment, type `python3 -m venv .venv`. The environment will be installed in the `.venv` folder in the root of the collection repo. This folder should not be committed. To enter the virtual environment once it has been created, run `source .venv/bin/activate`. Then run `pip3 install -r requirements.txt` to install the dependencies. These two steps are necessary to get the `post_omeka` and `post_omeka_html` scripts to run. +#### If you are working with an existing data repository -If running the script results in an error that a dependency is missing (i.e. `ModuleNotFound`) run `pip3 install [dependency]`. (It may be necessary to do an Internet search to determine the name of the needed package, which may differ between the `import` statement and the `pip3 install` command; e.g. `import dotenv` but `pip3 install python-dotenv`). After installing all necessary dependencies, you can run `pip3 freeze > requirements.txt`, and commit the `requirements.txt` file within the data repo. +In your Gemfile, make sure that Datura is on the right branch for Omeka posting. This functionality should be included in any recent release. Change `.ruby-gemset` to `datura-omeka` or something similar if this is not the version of Datura you usually use. Run `cd .` and then `bundle install`. If you plan to create overrides for any Omeka fields, you can copy the `lib/datura/python/omeka_overrides_example.py` in the Datura library (at `/lib/datura/python/`) to your repository as `[collection-directory]/scripts/python/omeka_overrides.py`. -### Config for Omeka S posting +### Step 2: Enable a virtual environment -The following settings should be placed in `config/private.yml` (in addition to the config that is already included for Datura): +In your collection repo, first exit any virtual environment if one is currently enabled (this may be indicated by `(.venv)` or similar text before your command prompt) with `deactivate`. If you have not previously created a virtual environment, type `python3 -m venv .venv`. The environment will be installed in the `.venv` folder in the root of the collection repo. This folder should not be committed. If you are working in a newly created repo, it should already be added to the `.gitignore` file. If you are working with an existing data repository, you may have to add the `.venv` directory to `.gitignore`. + +To enter the virtual environment once it has been created, run `source .venv/bin/activate`. + +### Step 3: Install Python dependencies + +Next, confirm you have a `requirements.txt` file in the root directory of your collection. If you are working with an existing repository, you may need to copy this over from Datura. Then, to install the dependencies, run: + +```bash +pip3 install packaging +pip3 install -r requirements.txt +``` + +The `packaging` library will need to be installed separately so the `omeka_s_tools` installation (part of the `requirements.txt` list) will work. + +### Step 4: Set up config for Omeka S posting + +If you have a newly created repo, you should see some omeka-related config values in your auto-generated `config/private.yml` config file. Uncomment these and fill in the values. If you are working with an existing repo, the following settings should be placed in `config/private.yml` (in addition to the config that is already included for Datura): ```yaml default: @@ -21,13 +38,16 @@ default: key_credential: ***** resource_template: ## omeka_data_base: desired/base/url/for/tei/files - iiif_server: servername.unl.edu + iiif_server: servername.unl.edu # optional, if the collection uses the image server + iiif_collection: collection_name # optional, if the collection uses the image server development: item_set: ## -production +production: item_set: ## ``` +All values not listed as optional are required for the omeka scripts to run. + - (for developers) `json_dir`, `html_id`, and `iiif_dir` are set within the script and correspond to the standard Datura output folders. The `key_identity` and `key_credential` fields should correspond to the generated API key credentials. which you can generate on your Omeka S user page (click "Edit user" and then the API key). Make sure to copy the credentials down right away after generating the key. @@ -36,8 +56,10 @@ Make sure that config is pointing to the right `resource_template` for the data `omeka_data_base` is necessary to indicate the URL to the TEI data documents. It should have a format like `https://github.com/CDRH/[repo_name]/blob/[env]/source/tei` or specify a similar relative path. The Omeka script adds the filename at the end. Make sure you have the right repo to make this a valid url. -For HTML posting, set `iiif_server` to the base url of the IIIF image server. +For media posting, set `iiif_server` to the base url of the IIIF image server and `iiif_collection` to the name of the collection or whatever name is used for the collection's iiif directory on the image server. + +`item_set` should be specified by environment in `private.yml` in order to categorize items by environment on Omeka S. The proper item_set id can be found in Omeka if you append `admin/item-set` to the base Omeka site URL. Look for `Environment--Development` or something similar; the id will appear at the end of the URL if you click the link. Not all projects have environments. -`item_set` should be specified by environment in `private.yml` in order to categorize items by environment on Omeka S. The proper item_set id can be found in Omeka if you append `admin/item-set` to the base Omeka site URL. Look for `Environment--Development` or something similar; the id will appear at the end of the URL if you click the link. Not all projects have environments and specifying an item set is not necessary to post. +### Step 5: Prepare to post! -See [post_omeka instructions](../3_manage/post_omeka.md) and [post_omeka_html instructions](../3_manage/post_omeka_html.md) for more information. \ No newline at end of file +See [post_omeka instructions](../3_manage/post_omeka.md) and [post_omeka_html instructions](../3_manage/post_omeka_html.md) for information about posting to Omeka S. \ No newline at end of file From 02525f5e81b4f18d22716a563213e3543400e37f Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 15:36:01 -0500 Subject: [PATCH 126/222] cleanup docs --- docs/1_setup/omeka_setup.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/1_setup/omeka_setup.md b/docs/1_setup/omeka_setup.md index bb06dafd6..3c85b3b4d 100644 --- a/docs/1_setup/omeka_setup.md +++ b/docs/1_setup/omeka_setup.md @@ -1,6 +1,6 @@ ## Set up for Omeka S posting -### Step 1: Setting up a data repository for Omeka +### Step 1: Set up a data repository for Omeka #### If you would like to create a new repository @@ -8,13 +8,30 @@ Follow [the steps](https://github.com/CDRH/datura/blob/dev/docs/1_setup/collecti #### If you are working with an existing data repository -In your Gemfile, make sure that Datura is on the right branch for Omeka posting. This functionality should be included in any recent release. Change `.ruby-gemset` to `datura-omeka` or something similar if this is not the version of Datura you usually use. Run `cd .` and then `bundle install`. If you plan to create overrides for any Omeka fields, you can copy the `lib/datura/python/omeka_overrides_example.py` in the Datura library (at `/lib/datura/python/`) to your repository as `[collection-directory]/scripts/python/omeka_overrides.py`. +In your Gemfile, make sure that Datura is on the right branch for Omeka posting. This functionality should be included in any recent release. Change `.ruby-gemset` to `datura-omeka` or something similar if this is not the version of Datura you usually use. Run: + +```bash +cd . +bundle install +``` + +If you plan to create overrides for any Omeka fields, you can copy the `lib/datura/python/omeka_overrides_example.py` in the Datura library (at `/lib/datura/python/`) to your repository as `[collection-directory]/scripts/python/omeka_overrides.py`. ### Step 2: Enable a virtual environment -In your collection repo, first exit any virtual environment if one is currently enabled (this may be indicated by `(.venv)` or similar text before your command prompt) with `deactivate`. If you have not previously created a virtual environment, type `python3 -m venv .venv`. The environment will be installed in the `.venv` folder in the root of the collection repo. This folder should not be committed. If you are working in a newly created repo, it should already be added to the `.gitignore` file. If you are working with an existing data repository, you may have to add the `.venv` directory to `.gitignore`. +In your collection repo, first exit any virtual environment if one is currently enabled (this may be indicated by `(.venv)` or similar text before your command prompt) with `deactivate`. If you have not previously created a virtual environment, run: + +```bash +python3 -m venv .venv +``` + +The environment will be installed in the `.venv` folder in the root of the collection repo. This folder should not be committed. If you are working in a newly created repo, it should already be added to the `.gitignore` file. If you are working with an existing data repository, you may have to add the `.venv` directory to `.gitignore`. + +To enter the virtual environment once it has been created, run -To enter the virtual environment once it has been created, run `source .venv/bin/activate`. +```bash +source .venv/bin/activate +``` ### Step 3: Install Python dependencies From 3547e666e73fccc916abf92850f6832a589aad80 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 16:07:36 -0500 Subject: [PATCH 127/222] update overrides and posting docs, minor adjustments to setup docs --- docs/1_setup/omeka_setup.md | 6 ++++-- docs/2_customization/omeka_overrides.md | 8 ++++---- docs/3_manage/post_omeka.md | 10 +++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/1_setup/omeka_setup.md b/docs/1_setup/omeka_setup.md index 3c85b3b4d..30ef8516b 100644 --- a/docs/1_setup/omeka_setup.md +++ b/docs/1_setup/omeka_setup.md @@ -33,6 +33,8 @@ To enter the virtual environment once it has been created, run source .venv/bin/activate ``` +You should now see a `(.venv)` at the front of the command line prompt. You will need to activate this environment every time you post to Omeka S. + ### Step 3: Install Python dependencies Next, confirm you have a `requirements.txt` file in the root directory of your collection. If you are working with an existing repository, you may need to copy this over from Datura. Then, to install the dependencies, run: @@ -42,7 +44,7 @@ pip3 install packaging pip3 install -r requirements.txt ``` -The `packaging` library will need to be installed separately so the `omeka_s_tools` installation (part of the `requirements.txt` list) will work. +The `packaging` library will need to be installed separately so the `omeka_s_tools` installation (part of the `requirements.txt` list) will install correctly. ### Step 4: Set up config for Omeka S posting @@ -65,7 +67,7 @@ production: All values not listed as optional are required for the omeka scripts to run. -- (for developers) `json_dir`, `html_id`, and `iiif_dir` are set within the script and correspond to the standard Datura output folders. +- (for developers) `json_dir`, `html_id`, and `iiif_dir` are set within the Omeka S scripts and correspond to the standard Datura output folders. The `key_identity` and `key_credential` fields should correspond to the generated API key credentials. which you can generate on your Omeka S user page (click "Edit user" and then the API key). Make sure to copy the credentials down right away after generating the key. diff --git a/docs/2_customization/omeka_overrides.md b/docs/2_customization/omeka_overrides.md index 3d6226a71..3274ccdc8 100644 --- a/docs/2_customization/omeka_overrides.md +++ b/docs/2_customization/omeka_overrides.md @@ -2,13 +2,13 @@ ### Standard definitions of fields -Each Omeka field is updated by the method in [api_fields.py](../../../lib/datura/python/api_fields.py) to compile the Omeka S JSON. This method takes the form `update_item_value(item, key, value, datatype="literal")`, the first argumment is the json hash with the API data, the second argument corresponds to the field in the resource template, and the third is the return value the corresponding function of `field_definitions.py`. Optionally, you can pass in the datatype, as the fourth argument. The default definitions of Omeka fields are in [field_definitions.py](../../../lib/datura/python/field_definitions.py). The Omeka API fields defined here must correspond with the Omeka resource template you are using, and the return value should be compatible with the data type.If you do not specify it, it will be set to "literal". For example `update_item_value(built_item, "dcterms:date", fields.date(json), "numeric:timestamp")`. +Each Omeka field is updated by the method in [api_fields.py](../../../lib/datura/python/api_fields.py) to compile the Omeka S JSON. This method takes the form `update_item_value(item, key, value, datatype="literal")`, the first argumment is the json hash with the API data, the second argument corresponds to the field in the resource template, and the third is the return value the corresponding function of `field_definitions.py`. Optionally, you can pass in the datatype, as the fourth argument. The default definitions of Omeka fields are in [field_definitions.py](../../../lib/datura/python/field_definitions.py). The Omeka API fields defined here must correspond with the Omeka resource template you are using, and the return value should be compatible with the data type. If you do not specify it, it will be set to "literal". For example `update_item_value(built_item, "dcterms:date", fields.date(json), "numeric:timestamp")`. ### Overriding fields -To override the field definitions, copy the file [omeka_overrides_example.py](../../../lib/datura/python/omeka_overrides_example.py) to [omeka_overrides.py](../../../lib/datura/python/omeka_overrides.py) in the `scripts/overrides` file of the project directory. Then override each method as needed, using the existing definitions in `field_definitions.py` as examples. For an example, see https://github.com/CDRH/data_stories_humanity/blob/omeka_s_ingest/scripts/python/omeka_overrides.py (not currently field). +To override the field definitions, copy the file [omeka_overrides_example.py](../../../lib/datura/python/omeka_overrides_example.py) to [omeka_overrides.py](../../../lib/datura/python/omeka_overrides.py) in the `scripts/python` file of the project directory. Then override each method as needed, using the commented patterns in the example overrides file as a guide. -Each overriden method needs to take the arguments `self` (a Python placeholder for a class instance) and `json` (representing the generated JSON) and to match the methods defined on `field_definitions.py`. (The same goes for adding new methods to `field_definitions.py`.) +Each overriden method needs to take the arguments `self` (a Python placeholder for a class instance) and `json` (representing the generated JSON) and to match the methods defined in `field_definitions.py`. (The same goes for adding new methods to `field_definitions.py`.) For instance: ```python @@ -20,7 +20,7 @@ For instance: return person_names ``` -First retrieve the value from the Elasticsearch `json` (keeping in mind that it is sometimes nil), then do any manipulations needed before returning the desired value. The return value must be either an list or single value. For single values, usually this will be the same as the value in the JSON. But Unlike the Elasticsearch-based API, it is not possible to ingest nested fields into Omeka S, so they must be reduced into array form. See [field_definitions.py](../../../lib/datura/python/field_definitions.py)for examples of how to retrieve single and nested values from the JSON, manipulate them and return the proper values for Omeka S. +First retrieve the value from the Elasticsearch `json` (keeping in mind that it is sometimes nil), then do any manipulations needed before returning the desired value. The return value must be either an list or single value. For single values, usually this will be the same as the value in the JSON. But Unlike the ElasticSearch-based API, it is not possible to ingest nested fields into Omeka S, so they must be reduced into array form. See [field_definitions.py](../../../lib/datura/python/field_definitions.py)for examples of how to retrieve single and nested values from the JSON, manipulate them and return the proper values for Omeka S. ### Linking items diff --git a/docs/3_manage/post_omeka.md b/docs/3_manage/post_omeka.md index 9c6ec706f..c7309aefc 100644 --- a/docs/3_manage/post_omeka.md +++ b/docs/3_manage/post_omeka.md @@ -1,12 +1,12 @@ ## Instructions for posting data into Omeka API -See (omeka setup instructions)[../1_setup/omeka_setup.md] for how to prepare your repo, config, and activate the Python virtual environment. +See (omeka setup instructions)[../1_setup/omeka_setup.md] for how to prepare and configure your data repository and activate the Python virtual environment. Running the `post_omeka` script will first run the Datura scripts to generate JSON files with the standard fields and values of the CDRH API (this is what is normally sent to Elasticsearch when you run `post`). This first step is equivalent to running `post -x es -o -t`. It then sends the generated JSON to the Python scripts to be ingested into Omeka S. Use the `-s` option to skip the generation step and only post to Omeka S (requires that you have already generated the needed documents by running `post_omeka` normally). -It is possible to run `post_omeka` with Datura's other command line options as described in [post.md] (for instance `-f` to filter by file type and `-r` and filter by regex), but it is not recommended to override the default options such as `-x es` +It is possible to run `post_omeka` with Datura's other command line options as described in [post.md] (for instance `-f` to filter by format and `-r` to filter by regex), but it is not recommended to override the default options such as `-x es`. You can specify the environment with `-e [environment]` but you must set an `item_set` with the desired environment in `config/private.yml.` See (omeka setup instructions)[../1_setup/omeka_setup.md] for more details. @@ -14,9 +14,9 @@ For information on how to override field definitions, see [Omeka Overrides](../2 ## Troubleshooting -### notes on debugging +### Notes on debugging -The standard way to debug Python scripts is with `breakpoint()`, equivalent to `byebug` in Ruby. Execution will pause and then you can check the contents of variables and try snippets of code from the prompt. Sometimes putting a debugger within an except clause (especially within a loop) makes it difficult to break out of the script and halt execution even if you type `quit`. CTRL-C sometimes works in these cases. If CTRL-C also fails to halt the script, try running `os._exit(0)` (this is the reason for `import os` in some of the scripts, and if you get an error that the module is not found, then `import os` should be run from the debugger prompt first). For more details on a particular error, look at the error message which is printed in some of the except clauses, check online documentation to see if this error message has additional methods (depending on the error), or use `traceback.print_exc()` to print out the full stack trace. In the case of errors in the HTTP reponse, it may be necessary to look in the logs on the Omeka site. +The standard way to debug Python scripts is with `breakpoint()`, equivalent to `byebug` in Ruby. Execution will pause and then you can check the contents of variables and try snippets of code from the prompt. Sometimes putting a debugger within an except clause (especially within a loop) makes it difficult to break out of the script and halt execution even if you type `quit`. CTRL-C sometimes works in these cases. If CTRL-C also fails to halt the script, try running `os._exit(0)` (this is the reason for `import os` in some of the scripts, and if you get an error that the module is not found, then `import os` should be run from the debugger prompt first). For more details on a particular error, look at the error message which is printed in some of the except clauses, check online documentation to see if this error message has additional methods (depending on the error), or use `traceback.print_exc()` to print out the full stack trace. You can also check the logs at `/logs/python.log`. In the case of errors in the HTTP reponse, it may be necessary to look in the logs on the Omeka site. ### 500 error @@ -24,7 +24,7 @@ Look in the error log on the Omeka site. This may indicate a configuration probl ### Data type not allowed in template -Check your script in api_fields.py to make sure you are passing the right data types, as specified in the resource template you are using. The default data type is "literal". If the resource template changes you must change the data types in your script, too. +Check your script in `api_fields.py` to make sure you are passing the right data types, as specified in the resource template you are using. The default data type is "literal". If the resource template changes you must change the data types in your script, too. ### Term not in template From 7c7a29aceb0b84b927328ba738299b494d274190 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 16:12:14 -0500 Subject: [PATCH 128/222] update examples in docs, remove git-ignore note --- docs/3_manage/post.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index f9b2dd4b8..6ea4eaed0 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -84,11 +84,11 @@ Outputs transformed files to a collection's `output/[environment]/[type]`. This Proceed with posting from (and including) the file matching this regex. Files are sorted alphabetically before the proceed point is located. The regex must match exactly one file or the script will exit with an error. Can be combined with `-r` to proceed within a filtered set. -Example: `post -R let0050` (post all files from `let0050` onward) +Example: `post -p let0050` (post all files from `let0050` onward) -Example: `post -r let -R let0050` (post all `let` files from `let0050` onward) +Example: `post -r let -p let0050` (post all `let` files from `let0050` onward) -**Checkpoint file**: After each batch of files is posted (not in `--transform-only` mode), Datura writes the last posted filename to `logs/proceed_{environment}` in your collection directory. Add `logs/proceed_*` to your collection's `.gitignore` to avoid committing this file. +**Checkpoint file**: After each batch of files is posted (not in `--transform-only` mode), Datura writes the last posted filename to `logs/proceed_{environment}` in your collection directory. **Interactive resume**: When `-p` is given with no value (`post -p`), Datura reads the checkpoint file and prompts: From 115a3884afb6c43ee7f3b5918c02634f5f17b04d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 16:26:28 -0500 Subject: [PATCH 129/222] fix typo and indentation --- lib/datura/data_manager.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 03d4cbd20..c7c134a3a 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -129,10 +129,10 @@ def batch_process_files end # wait for all the files to process before moving on with the next chunk threads.each { |t| t.join } - # save checkpoint after chunk completes (not in transform-only mode) - unless @options["transform_only"] - Datura::Helpers.write_checkpoint(files_subset.last.filename(false), @options) - end + # save checkpoint after chunk completes (not in transform-only mode) + unless @options["transform_only"] + Datura::Helpers.write_checkpoint(files_subset.last.filename(false), @options) + end end end @@ -286,7 +286,7 @@ def prepare_files puts msg.yellow @log.warn(msg) end - # prcoeed from (and including) a specific file + # proceed from (and including) a specific file proceeded = if @options["proceed"] Datura::Helpers.proceed_files(regexed, @options["proceed"]) else From d939f752b104494bc619017d69e475ac8ea378e2 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 5 Jun 2026 16:40:28 -0500 Subject: [PATCH 130/222] add conditional so last file is not written to checkpoint if posting finishes --- lib/datura/data_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index c7c134a3a..d512e4fb6 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -130,7 +130,7 @@ def batch_process_files # wait for all the files to process before moving on with the next chunk threads.each { |t| t.join } # save checkpoint after chunk completes (not in transform-only mode) - unless @options["transform_only"] + unless @options["transform_only"] || files_subset.last == @files.last Datura::Helpers.write_checkpoint(files_subset.last.filename(false), @options) end end From 7e3e09a86fabfbe6cb8e7b2029614a6a457cb008 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 11:01:35 -0500 Subject: [PATCH 131/222] add clear checkpoint helper --- lib/datura/helpers.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index d35904384..a50e6c72d 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -173,6 +173,15 @@ def self.write_checkpoint(basename, options) File.write(path, "#{basename}\n") end + # clear_checkpoint + # writes empty content to the checkpoint file, signaling no resume point + # params: options (hash) + # returns: nil + def self.clear_checkpoint(options) + path = checkpoint_path(options) + File.write(path, "") + end + # should_update? # determines if a user has changed a file since specified date # params: file (string path), since_date (Time format or nil) From d3662b7927215948939e7fe06a81c5b6d9e85c4a Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 11:12:07 -0500 Subject: [PATCH 132/222] fix checkpoint handling: remove -t limit, add branch for process sort, adjust last file call --- lib/datura/data_manager.rb | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index d512e4fb6..d14be47b3 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -1,7 +1,7 @@ require "colorize" require "logger" require "yaml" -require "byebug" + require_relative "./requirer.rb" class Datura::DataManager @@ -129,10 +129,24 @@ def batch_process_files end # wait for all the files to process before moving on with the next chunk threads.each { |t| t.join } - # save checkpoint after chunk completes (not in transform-only mode) - unless @options["transform_only"] || files_subset.last == @files.last - Datura::Helpers.write_checkpoint(files_subset.last.filename(false), @options) + # save checkpoint after chunk completes + Datura::Helpers.write_checkpoint(files_subset.last.filename(false), @options) + end + # clear checkpoint if all files in the source directories were posted (not a filtered subset) + unless @files.empty? + last_overall = if @options["proceed"] + # proceed_files sorts all files alphabetically across directories + Datura::DataManager.format_to_class.keys.flat_map { |fmt| + Datura::Helpers.get_directory_files(File.join(@options["collection_dir"], "source", fmt)) || [] + }.map { |f| File.basename(f, ".*") }.sort.last + else + # normal run processes files in format_to_class.keys directory order + Datura::DataManager.format_to_class.keys.filter_map { |fmt| + found = Datura::Helpers.get_directory_files(File.join(@options["collection_dir"], "source", fmt)) + found&.map { |f| File.basename(f, ".*") }&.sort&.last + }.last end + Datura::Helpers.clear_checkpoint(@options) if @files.last.filename(false) == last_overall end end From 2d6b01f10a058a39958f60f27b7ce8702f3676b0 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 11:37:19 -0500 Subject: [PATCH 133/222] align proceed with post sort (directory-based) --- lib/datura/data_manager.rb | 16 ++++------------ lib/datura/helpers.rb | 12 ++++++++---- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index d14be47b3..092d84476 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -134,18 +134,10 @@ def batch_process_files end # clear checkpoint if all files in the source directories were posted (not a filtered subset) unless @files.empty? - last_overall = if @options["proceed"] - # proceed_files sorts all files alphabetically across directories - Datura::DataManager.format_to_class.keys.flat_map { |fmt| - Datura::Helpers.get_directory_files(File.join(@options["collection_dir"], "source", fmt)) || [] - }.map { |f| File.basename(f, ".*") }.sort.last - else - # normal run processes files in format_to_class.keys directory order - Datura::DataManager.format_to_class.keys.filter_map { |fmt| - found = Datura::Helpers.get_directory_files(File.join(@options["collection_dir"], "source", fmt)) - found&.map { |f| File.basename(f, ".*") }&.sort&.last - }.last - end + last_overall = Datura::DataManager.format_to_class.keys.filter_map { |fmt| + found = Datura::Helpers.get_directory_files(File.join(@options["collection_dir"], "source", fmt)) + found&.map { |f| File.basename(f, ".*") }&.sort&.last + }.last Datura::Helpers.clear_checkpoint(@options) if @files.last.filename(false) == last_overall end end diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index a50e6c72d..6c04e3a92 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -122,13 +122,17 @@ def self.regex_files(files, regex=nil) end # proceed_files - # sorts files alphabetically and returns all files from the first file - # matching the proceed regex onward (inclusive). Exits with an error if - # the regex matches zero or more than one file. + # returns all files from the first file matching the proceed regex onward (inclusive), + # preserving directory order and sorting alphabetically within each directory. + # Exits with an error if the regex matches zero or more than one file. # params: files (array of file paths), regex (string) # returns: array def self.proceed_files(files, regex) - sorted = files.sort_by { |f| File.basename(f, ".*") } + # Preserve directory order from input list; sort alphabetically within each directory + dir_order = files.map { |f| File.dirname(f) }.uniq + sorted = dir_order.flat_map do |dir| + files.select { |f| File.dirname(f) == dir }.sort_by { |f| File.basename(f, ".*") } + end exp = validate_regex(regex, "--proceed") matches = sorted.select { |f| exp.match(File.basename(f, ".*")) } From 20d77f785e2eb854c4d1fe77aa454021e6023ed4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 11:43:12 -0500 Subject: [PATCH 134/222] fix spacing --- lib/datura/data_manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 092d84476..2d81e278e 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -134,7 +134,7 @@ def batch_process_files end # clear checkpoint if all files in the source directories were posted (not a filtered subset) unless @files.empty? - last_overall = Datura::DataManager.format_to_class.keys.filter_map { |fmt| + last_overall = Datura::DataManager.format_to_class.keys.filter_map { |fmt| found = Datura::Helpers.get_directory_files(File.join(@options["collection_dir"], "source", fmt)) found&.map { |f| File.basename(f, ".*") }&.sort&.last }.last From 560db41504deaf45875dd6340928a25927d404ec Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 13:19:03 -0500 Subject: [PATCH 135/222] add omeka options to main post list so all are accounted for somewhere --- docs/3_manage/post.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index 2a013f4c5..dae7d5920 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -54,6 +54,13 @@ Format options include: If you do not select any, all the formats found will be executed. +```bash +-m, --no-media +``` + +Omeka specific, this will skip the step of deleting and regenerating media if both HTML and cover image are present in Omeka S. + + ```bash -n, --no-commit ``` @@ -78,6 +85,12 @@ Transforms / posts only files matching a specific regular expression. DO NOT in Example: `post -r let0001` +```bash +-s, --skip +``` + +Omeka specific, this skips the JSON or HTML generation step and only posts to Omeka S (it can be used with both `post_omeka` and `post_omeka_html`). + ```bash -t, --transform-only ``` @@ -127,4 +140,4 @@ If you get an error when you run `bundle install`, try running the suggested com ### Posting to Omeka -For posting to Omeka, see [post_omeka.md] and [post_omeka_html.md] +For posting to Omeka, see [post_omeka.md](post_omeka.md) and [post_omeka_html.md](post_omeka_html.md) From 1242454c32e5ed9e563bbc4e6a7cc5e816f536e9 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 14:17:50 -0500 Subject: [PATCH 136/222] add omeka json output option --- bin/post_omeka | 6 ++++ docs/3_manage/post.md | 8 +++++ lib/datura/helpers.rb | 1 + lib/datura/python/json_to_omeka.py | 49 ++++++++++++++++++++++++++++-- 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index b35fad137..ee8e16edb 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -27,6 +27,10 @@ optparse = OptionParser.new do |opts| end end + opts.on('-j', '--json-output', 'write Omeka S item payloads to output//omeka/ instead of posting to the API (API connection still required for property ID lookups)') do + options["json_output"] = true + end + opts.on('-r', '--regex [input]', 'only generate and post files matching this regex') do |input| if input && input.length > 0 options["regex"] = input @@ -42,6 +46,8 @@ optparse.parse(ARGV) #remove options not used in the main script ARGV.delete("-s") ARGV.delete("--skip") +ARGV.delete("-j") +ARGV.delete("--json-output") #add options to output a json file instead of posting it to Elasticsearch ARGV.unshift("-x", "es", "-o", "-t") #create DataManager before conditional run diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index dae7d5920..624778a10 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -54,6 +54,14 @@ Format options include: If you do not select any, all the formats found will be executed. +```bash +-j, --json-output +``` + +*Default setting: false* + +Omeka specific, this writes Omeka S item payloads to `output//omeka/` instead of posting to the API. This is useful for debugging and inspection purposes. + ```bash -m, --no-media ``` diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 07c364869..9ea0eb88c 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -166,6 +166,7 @@ def self.run_omeka_script(script_path, options) command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] command.append("-f", Shellwords.escape(options["format"])) if options["format"] command.append("-m") if options["media_skip"] + command.append("-j") if options["json_output"] command.append("--error-file", error_file_path) system(*command) omeka_errors = begin diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index d373b8343..c49a9f8de 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -88,6 +88,19 @@ def _parse_args(): dest="format_filter", help="Only post files of this format (tei, csv, vra, ead, html, pdf, webs).", ) + parser.add_argument( + "-j", "--json-output", + action="store_true", + dest="json_output", + default=False, + help=( + "Write Omeka S item payloads to output//omeka/ instead " + "of posting to the API. No items are created or updated in Omeka. " + "An API connection is still required for property ID lookups and " + "template validation. The link pass is skipped because no live " + "Omeka item IDs are available." + ), + ) parser.add_argument( "-r", "--regex", default=None, @@ -127,7 +140,7 @@ def _parse_args(): # Pass 1: item creation / update # --------------------------------------------------------------------------- -def post_items(ctx, pathlist): +def post_items(ctx, pathlist, json_output_dir=None): """ First pass: create or update Omeka items for every JSON record. @@ -144,8 +157,10 @@ def post_items(ctx, pathlist): credentials, missing config) raise exceptions that propagate to main(). Parameters: - * ctx - OmekaContext providing API client, config, and error log - * pathlist - list of pathlib.Path objects pointing to ES JSON files + * ctx - OmekaContext providing API client, config, and error log + * pathlist - list of pathlib.Path objects pointing to ES JSON files + * json_output_dir - optional Path; when set, write Omeka S item payloads to + this directory instead of posting to the API """ for path in pathlist: filename = str(path) @@ -163,6 +178,21 @@ def post_items(ctx, pathlist): logger.warning("Skipping item without identifier in %s", filename) continue + if json_output_dir is not None: + # JSON output mode: build the payload and write it to disk + # rather than to the Omeka API. + new_item = api_fields.prepare_item(ctx, json_item) + if not new_item: + logger.warning("Could not prepare payload for %r; skipping", identifier) + continue + payload = prepare_item_payload_using_template(ctx, new_item, template_number) + out_path = json_output_dir / "{}.json".format(identifier) + relative_path = "output/{}/{}.json".format(ctx.environment, identifier) + logger.info("Writing Omeka payload for %r to %s", identifier, relative_path) + with open(out_path, "w") as f: + json.dump(payload, f, indent=2) + continue + try: matching_items = ctx.client.filter_items_by_property( filter_property="dcterms:identifier", @@ -446,6 +476,19 @@ def main(): ctx.environment, ) + # --- JSON output mode (-j / --json-output) --- + if args.json_output: + relative_dir = "output/{}/omeka".format(ctx.environment) + omeka_out_dir = ctx.resolve_path(relative_dir) + Path(omeka_out_dir).mkdir(parents=True, exist_ok=True) + logger.info( + "JSON output mode: writing Omeka S payloads to %s (API will not be called)", + relative_dir, + ) + post_items(ctx, pathlist, json_output_dir=Path(omeka_out_dir)) + finish_run(ctx, args, start_time) + return + # --- Pass 1: create / update items --- logger.info("Starting pass 1: item posting") post_items(ctx, pathlist) From c242bdbfe5a4155c88a8e64fee10a870a7390f1b Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 15:24:57 -0500 Subject: [PATCH 137/222] update omeka docs w/r/t restructuring --- docs/2_customization/omeka_overrides.md | 8 ++++---- docs/3_manage/post_omeka.md | 4 ++-- docs/3_manage/post_omeka_html.md | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/2_customization/omeka_overrides.md b/docs/2_customization/omeka_overrides.md index 3274ccdc8..24ac84134 100644 --- a/docs/2_customization/omeka_overrides.md +++ b/docs/2_customization/omeka_overrides.md @@ -2,13 +2,13 @@ ### Standard definitions of fields -Each Omeka field is updated by the method in [api_fields.py](../../../lib/datura/python/api_fields.py) to compile the Omeka S JSON. This method takes the form `update_item_value(item, key, value, datatype="literal")`, the first argumment is the json hash with the API data, the second argument corresponds to the field in the resource template, and the third is the return value the corresponding function of `field_definitions.py`. Optionally, you can pass in the datatype, as the fourth argument. The default definitions of Omeka fields are in [field_definitions.py](../../../lib/datura/python/field_definitions.py). The Omeka API fields defined here must correspond with the Omeka resource template you are using, and the return value should be compatible with the data type. If you do not specify it, it will be set to "literal". For example `update_item_value(built_item, "dcterms:date", fields.date(json), "numeric:timestamp")`. +Each Omeka field is updated by the method in [api_fields.py](../../../lib/datura/python/api_fields.py) to compile the Omeka S JSON. This method takes the form `update_item_value(ctx, item, key, value, datatype="literal")`. The first argument is the context parameter (providing access to config values, OmekaAPIClient, and property ID cache), the second argument is the json hash with the API data, the third argument corresponds to the field in the resource template, and the fourth is the return value of the corresponding function of `field_definitions.py`. Optionally, you can pass in the datatype, as the fifth argument. The default definitions of Omeka fields are in [field_definitions.py](../../../lib/datura/python/field_definitions.py). The Omeka API fields defined here must correspond with the Omeka resource template you are using, and the return value should be compatible with the data type. If you do not specify it, it will be set to "literal". For example `update_item_value(ctx, built_item, "dcterms:date", fields.date(json), "numeric:timestamp")`. ### Overriding fields To override the field definitions, copy the file [omeka_overrides_example.py](../../../lib/datura/python/omeka_overrides_example.py) to [omeka_overrides.py](../../../lib/datura/python/omeka_overrides.py) in the `scripts/python` file of the project directory. Then override each method as needed, using the commented patterns in the example overrides file as a guide. -Each overriden method needs to take the arguments `self` (a Python placeholder for a class instance) and `json` (representing the generated JSON) and to match the methods defined in `field_definitions.py`. (The same goes for adding new methods to `field_definitions.py`.) +Each overridden method needs to take the arguments `self` (a Python placeholder for a class instance) and `json` (representing the generated JSON) and to match the methods defined in `field_definitions.py`. (The same goes for adding new methods to `field_definitions.py`.) For instance: ```python @@ -20,7 +20,7 @@ For instance: return person_names ``` -First retrieve the value from the Elasticsearch `json` (keeping in mind that it is sometimes nil), then do any manipulations needed before returning the desired value. The return value must be either an list or single value. For single values, usually this will be the same as the value in the JSON. But Unlike the ElasticSearch-based API, it is not possible to ingest nested fields into Omeka S, so they must be reduced into array form. See [field_definitions.py](../../../lib/datura/python/field_definitions.py)for examples of how to retrieve single and nested values from the JSON, manipulate them and return the proper values for Omeka S. +First retrieve the value from the Elasticsearch `json` (keeping in mind that it is sometimes nil), then do any manipulations needed before returning the desired value. The return value must be either an list or single value. For single values, usually this will be the same as the value in the JSON. But unlike the ElasticSearch-based API, it is not possible to ingest nested fields into Omeka S, so they must be reduced into array form. See [field_definitions.py](../../../lib/datura/python/field_definitions.py)for examples of how to retrieve single and nested values from the JSON, manipulate them and return the proper values for Omeka S. ### Linking items @@ -29,7 +29,7 @@ Any new fields that link to the id of another item should be added to the `link_ ```python try: part_ids = [part['id'] for part in json_item["has_part"]] - link_item_record(existing_item, "dcterms:hasPart", part_ids) + link_item_record(ctx, existing_item, "dcterms:hasPart", part_ids) except Exception: pass ``` diff --git a/docs/3_manage/post_omeka.md b/docs/3_manage/post_omeka.md index c7309aefc..d26907da0 100644 --- a/docs/3_manage/post_omeka.md +++ b/docs/3_manage/post_omeka.md @@ -2,7 +2,7 @@ See (omeka setup instructions)[../1_setup/omeka_setup.md] for how to prepare and configure your data repository and activate the Python virtual environment. -Running the `post_omeka` script will first run the Datura scripts to generate JSON files with the standard fields and values of the CDRH API (this is what is normally sent to Elasticsearch when you run `post`). This first step is equivalent to running `post -x es -o -t`. It then sends the generated JSON to the Python scripts to be ingested into Omeka S. +Running the `post_omeka` script will first run the Datura scripts to generate JSON files with the standard fields and values of the CDRH API (this is what is normally sent to ElasticSearch when you run `post`). This first step is equivalent to running `post -x es -o -t`. It then sends the generated JSON to the Python scripts to be ingested into Omeka S. Use the `-s` option to skip the generation step and only post to Omeka S (requires that you have already generated the needed documents by running `post_omeka` normally). @@ -16,7 +16,7 @@ For information on how to override field definitions, see [Omeka Overrides](../2 ### Notes on debugging -The standard way to debug Python scripts is with `breakpoint()`, equivalent to `byebug` in Ruby. Execution will pause and then you can check the contents of variables and try snippets of code from the prompt. Sometimes putting a debugger within an except clause (especially within a loop) makes it difficult to break out of the script and halt execution even if you type `quit`. CTRL-C sometimes works in these cases. If CTRL-C also fails to halt the script, try running `os._exit(0)` (this is the reason for `import os` in some of the scripts, and if you get an error that the module is not found, then `import os` should be run from the debugger prompt first). For more details on a particular error, look at the error message which is printed in some of the except clauses, check online documentation to see if this error message has additional methods (depending on the error), or use `traceback.print_exc()` to print out the full stack trace. You can also check the logs at `/logs/python.log`. In the case of errors in the HTTP reponse, it may be necessary to look in the logs on the Omeka site. +The standard way to debug Python scripts is with `breakpoint()`, equivalent to `byebug` in Ruby. Execution will pause and then you can check the contents of variables and try snippets of code from the prompt. Sometimes putting a debugger within an except clause (especially within a loop) makes it difficult to break out of the script and halt execution even if you type `quit`. `CTRL-C` sometimes works in these cases. If `CTRL-C` also fails to halt the script, try running `os._exit(0)` (this is the reason for `import os` in some of the scripts, and if you get an error that the module is not found, then `import os` should be run from the debugger prompt first). For more details on a particular error, look at the error message which is printed in some of the except clauses, check online documentation to see if this error message has additional methods (depending on the error), or use `traceback.print_exc()` to print out the full stack trace. You can also check the logs at `/logs/python.log`. In the case of errors in the HTTP reponse, it may be necessary to look in the logs on the Omeka site. ### 500 error diff --git a/docs/3_manage/post_omeka_html.md b/docs/3_manage/post_omeka_html.md index 3593f9501..1021e6359 100644 --- a/docs/3_manage/post_omeka_html.md +++ b/docs/3_manage/post_omeka_html.md @@ -6,7 +6,7 @@ Use the `-s` option to skip the generation step and only post to Omeka S (requir Use the `-m` option to skip the step of deleting and regenerating for media items that have already been ingested. (It will only do this if both the image and html are uploaded already). -You can specify the environment with `-e [environment]` but you must set an `item_set` with the desired environment in config/private.yml. See [post_omeka instructions](docs/3_manage/post_omeka.md) for instructions. +You can specify the environment with `-e [environment]` but you must set an `item_set` with the desired environment in `config/private.yml`. See [post_omeka instructions](docs/3_manage/post_omeka.md) for instructions. Instructions for setting up the Python virtual enviroment and config can be found in (omeka setup instructions)[../1_setup/omeka_setup.md]. @@ -14,14 +14,14 @@ For information on how to override field definitions, see [Omeka Overrides](../2 ### Media ingesters (for developers) -The media payload, set in `html_and_media_ingest.py`, must be structured in a specific way to add items. It is different in the case of html and iiif images. +The media payload, set in `html_and_media_ingest.py`, must be structured in a specific way to add items. It is different in the case of html and iiif images. The `is_public` field is set in `omeka_context.py` and is based on environment (`production = True`, everything else = `False`). For an html field: ```json { - "o:is_public": True, + "o:is_public": ctx.is_public, "data": { - "html": html_content + "html": html_content, }, "o:ingester": "html" } @@ -29,9 +29,9 @@ For an html field: For a file upload (i.e. to upload): ```json { - "o:is_public": True, + "o:is_public": ctx.is_public, "data": { - "upload": html_content + "upload": html_content, }, "o:ingester": "upload" } @@ -39,7 +39,7 @@ For a file upload (i.e. to upload): For posting to the IIIF ingester (not currently implemented): ```json { - "o:is_public": True, + "o:is_public": ctx.is_public, "data": { "upload": iiif_url }, @@ -47,13 +47,13 @@ For a file upload (i.e. to upload): } ``` The IIIF URL should be in the format https://servername/iiif/2/collection_name%2Fitem_id.jpg/info.json. It should not point to a specific image. - `o:source`, set in the `add_media_to_item` in omeka.py, either corersponds to the filename or to the remote path if the ingester requires a remote URL. + `o:source`, set in `add_media_to_item` in the `omeka_s_tools` library, either corresponds to the filename or to the remote path if the ingester requires a remote URL. ## Troubleshooting -Sometimes running the script will return error code 422 (unprocessable content) or error code 500. These error messages can be investigated in the Omeka S logs found on the admin page. To investigate the errors, you can check the stack traces in these logs against the Omeka S source code found in GitHub. (Note that the base Omeka code, powering the website, is in PHP). +Sometimes running the script will return error code 422 (unprocessable content) or error code 500. These error messages can be investigated in the Omeka S logs found on the admin page. To investigate the errors, you can check the stack traces in these logs against the Omeka S source code found in GitHub or the Python logs in `/logs/python`. (Note that the base Omeka code, powering the website, is in PHP). - The ingester expects a full URL, not a local file path, in `o:source`, which is set when you post media items. - A malformed URL may cause an error that the script is unable to connect to a server. Make sure that it includes slashes in the proper places. - Internal SQL errors are likely also caused by sending bad data, not corresponding to the designated format. - There is a known issue where Omeka S raises an error when deleting media items and unlinking them, even though the action is performed successfully. -- Forbidden (403) errors may indicate that API credentials are missing or incorrect \ No newline at end of file +- Forbidden (403) errors may indicate that API credentials are missing or incorrect. \ No newline at end of file From ab77a33d68f33995e694497f914789a88f71cdb4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 8 Jun 2026 16:32:41 -0500 Subject: [PATCH 138/222] comment out -01-01 handling so this can be done on case-by-case basis --- lib/datura/python/field_definitions.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index ec5398300..669724f60 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -74,12 +74,14 @@ def contributor(self, json): def date(self, json): date_to_parse = json.get("date", None) - if date_to_parse and "-01-01" in date_to_parse: - #dates are automatically converted by Datura to yyyy-01-01 if month and date are missing - #convert such dates back to yyyy for Omeka S (since it is allowed by the date parser) - return datetime.strptime(date_to_parse, "%Y-%m-%d").year - else: - return date_to_parse + return date_to_parse + # Uncomment the below and remove the above line if this fix is needed + # if date_to_parse and "-01-01" in date_to_parse: + # #dates are automatically converted by Datura to yyyy-01-01 if month and date are missing + # #convert such dates back to yyyy for Omeka S (since it is allowed by the date parser) + # return datetime.strptime(date_to_parse, "%Y-%m-%d").year + # else: + # return date_to_parse def dateYear(self, json): date_to_parse = json.get("date", None) From 2777216a12bd50555c1f89287451f1522e63222d Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 9 Jun 2026 14:40:46 -0500 Subject: [PATCH 139/222] add note about pip upgrade to docs --- docs/1_setup/omeka_setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/1_setup/omeka_setup.md b/docs/1_setup/omeka_setup.md index 30ef8516b..7db5ca967 100644 --- a/docs/1_setup/omeka_setup.md +++ b/docs/1_setup/omeka_setup.md @@ -44,7 +44,7 @@ pip3 install packaging pip3 install -r requirements.txt ``` -The `packaging` library will need to be installed separately so the `omeka_s_tools` installation (part of the `requirements.txt` list) will install correctly. +The `packaging` library will need to be installed separately so the `omeka_s_tools` installation (part of the `requirements.txt` list) will install correctly. If installation fails, `pip` may need to be upgraded (the error message should advise this). ### Step 4: Set up config for Omeka S posting From d96390ac3bc9b2286e57f33d17cd095aaf26a5c5 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 9 Jun 2026 16:06:26 -0500 Subject: [PATCH 140/222] refactor sorted in proceed_files to avoid excessive looping --- lib/datura/helpers.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 6c04e3a92..9d1b47704 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -129,10 +129,9 @@ def self.regex_files(files, regex=nil) # returns: array def self.proceed_files(files, regex) # Preserve directory order from input list; sort alphabetically within each directory - dir_order = files.map { |f| File.dirname(f) }.uniq - sorted = dir_order.flat_map do |dir| - files.select { |f| File.dirname(f) == dir }.sort_by { |f| File.basename(f, ".*") } - end + sorted = files.group_by { |f| File.dirname(f) } + .sort_by { |dir, _| dir } + .flat_map { |_, fs| fs.sort_by { |f| File.basename(f, ".*") } } exp = validate_regex(regex, "--proceed") matches = sorted.select { |f| exp.match(File.basename(f, ".*")) } From 1e7526dd09177f3dd675f148fa9eca357bd0056c Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 9 Jun 2026 16:10:26 -0500 Subject: [PATCH 141/222] fix spacing --- lib/datura/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 9d1b47704..d7dc71dec 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -130,8 +130,8 @@ def self.regex_files(files, regex=nil) def self.proceed_files(files, regex) # Preserve directory order from input list; sort alphabetically within each directory sorted = files.group_by { |f| File.dirname(f) } - .sort_by { |dir, _| dir } - .flat_map { |_, fs| fs.sort_by { |f| File.basename(f, ".*") } } + .sort_by { |dir, _| dir } + .flat_map { |_, fs| fs.sort_by { |f| File.basename(f, ".*") } } exp = validate_regex(regex, "--proceed") matches = sorted.select { |f| exp.match(File.basename(f, ".*")) } From 62985393b62147ab703b65a435384931f1f62e60 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 10:03:23 -0500 Subject: [PATCH 142/222] shift bundler dev dependency back to earlier constraint with explanation --- datura.gemspec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datura.gemspec b/datura.gemspec index fec7219e2..f4442aa27 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -62,7 +62,9 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "rest-client", "~> 2.1" spec.add_runtime_dependency "pdf-reader", "~> 2.15" spec.add_development_dependency "byebug", "~> 11.0" - spec.add_development_dependency "bundler", ">= 2.0" + # leaving this constraint as-is to avoid possible conflicts with + # later versions of bundler requiring Ruby > 3.1 + spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" end From 08e5970121ecc09409eca5d9b3344ca90777693d Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 10:12:26 -0500 Subject: [PATCH 143/222] remove unused imports --- lib/datura/python/field_definitions.py | 1 - lib/datura/python/json_to_omeka.py | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 669724f60..cbf421d8b 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -1,6 +1,5 @@ import logging import sys -import os from datetime import datetime logger = logging.getLogger(__name__) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index c49a9f8de..c70470f18 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -35,7 +35,6 @@ import os import sys import time -import traceback from pathlib import Path import api_fields From fc9a8bac863695b438a95bead8af7f4a450ccea7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 10:14:41 -0500 Subject: [PATCH 144/222] remove unused functions; csv is handled on Ruby side of datura --- lib/datura/python/api_fields.py | 108 -------------------------------- 1 file changed, 108 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 7d7900fa5..dfec7fdc4 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -226,39 +226,6 @@ def link_records(ctx, row, existing_item): return link_item(ctx, row, existing_item) -def get_json_value(row, name): - """ - Extract a value from a CSV-derived row dict, handling multiple encodings. - - Datura serialises multi-valued fields from CSV in two ways: - - JSON array strings: '["value1", "value2"]' - - Semicolon-delimited strings: 'value1;;;value2' - - Single values are returned as-is. Empty strings return the empty string. - - Parameters: - * row - dict representing one CSV row - * name - the field name to extract - - Returns a string, list of strings, or empty string. - """ - if len(row[name]) > 0: - if row[name].startswith('["'): - # Deserialise a JSON-encoded array. - try: - return json.loads(row[name]) - except json.JSONDecodeError: - logger.warning("Could not parse JSON value for field %r: %r", name, row[name]) - return row[name] - elif ";;;" in row[name]: - # Split a semicolon-delimited multi-value string. - return row[name].split(";;;") - else: - return row[name] - else: - return row[name] - - def update_item_value(ctx, item, key, value, datatype="literal"): """ Set or replace a property on an Omeka item dict. @@ -334,81 +301,6 @@ def add_formatted_value(ctx, item, key, value, datatype, label=""): return item - -def get_matching_ids_from_markdown(row, field): - """ - Extract CDRH identifier strings from a field containing markdown-formatted links. - - Markdown link format: [Display Name](identifier) - This function extracts only the identifier (the part in parentheses). - - Parameters: - * row - dict representing one Datura JSON item - * field - the field name containing markdown link strings - - Returns a list of identifier strings, or an empty list if the field is - absent or contains no valid links. - """ - if row[field]: - markdown_values = sorted(get_json_value(row, field)) - ids = [] - if markdown_values: - if isinstance(markdown_values,str): - match = re.search(r"\]\((.*)\)", markdown_values) - if match: - ids.append(match.group(1)) - else: - for value in markdown_values: - match = re.search(r"\]\((.*)\)", value) - if match: - ids.append(match.group(1)) - if len(ids) > 1: - # Remove empty strings that may result from links with no - # destination (e.g. "[Name]()"). - ids = list(filter(None, ids)) - return ids - else: - return [] - - -def get_matching_names_from_markdown(row, field): - """ - Extract display names from a field containing markdown-formatted links, - filtering out names that have a corresponding identifier. - - Markdown link format: [Display Name](identifier) - This function extracts only the display name (the part in brackets), but - skips entries where an identifier is also present, since those items can - be resolved by ID via get_matching_ids_from_markdown. - - Parameters: - * row - dict representing one Datura JSON item - * field - the field name containing markdown link strings - - Returns a list of display name strings, or an empty list if the field is - absent or all entries have identifiers. - """ - if row[field]: - markdown_values = get_json_value(row, field) - names = [] - if markdown_values: - if isinstance(markdown_values,str): - name_match = re.search(r"\[(.*?)\]", markdown_values) - id_match = re.search(r"\]\((.*)\)", markdown_values) - # Only collect the name if there is no associated identifier. - if name_match and (not id_match or not id_match.group(1)): - names.append(name_match.group(1)) - else: - for value in markdown_values: - name_match = re.search(r"\[(.*?)\]", value) - id_match = re.search(r"\]\((.*)\)", value) - if name_match and (not id_match or not id_match.group(1)): - names.append(name_match.group(1)) - return names - else: - return [] - - def get_omeka_ids(ctx, lookup_values, filter_property): """ Resolve a list of lookup values to Omeka numeric item IDs. From 740e718140251195429caae355450e12362870b7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 10:47:50 -0500 Subject: [PATCH 145/222] refactor to limit glob calls to one pass for filter_items_by_date and filter_items_by_format --- lib/datura/python/omeka.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 73033625c..01009f6f4 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -121,10 +121,15 @@ def filter_items_by_date(update_time, pathlist): * pathlist - iterable of pathlib.Path or string paths to filter """ source_base = Path.cwd() / "source" + # Build index once: stem -> list of source paths + source_index = {} + for sf in source_base.glob("*/*.*"): + source_index.setdefault(sf.stem, []).append(sf) + result = [] for p in pathlist: - identifier = Path(str(p)).stem - source_files = list(source_base.glob("*/{}.*".format(identifier))) + identifier = Path(p).stem + source_files = source_index.get(identifier, []) if not source_files: logger.debug( "No source file found for %r; including without date filter", identifier @@ -132,14 +137,13 @@ def filter_items_by_date(update_time, pathlist): result.append(p) continue source_mtime = max( - datetime.fromtimestamp(os.path.getmtime(str(sf))) for sf in source_files + datetime.fromtimestamp(sf.stat().st_mtime) for sf in source_files ) if source_mtime >= update_time: result.append(p) return result - def filter_items_by_format(format_type, pathlist): """ Filter a list of output JSON file paths to those whose source file lives @@ -159,8 +163,6 @@ def filter_items_by_format(format_type, pathlist): * pathlist - iterable of pathlib.Path or string paths to filter """ source_dir = Path.cwd() / "source" / format_type - result = [] - for p in pathlist: - if list(source_dir.glob("{}.*".format(Path(p).stem))): - result.append(p) - return result \ No newline at end of file + # Build the set of stems once instead of globbing per item + source_stems = {sf.stem for sf in source_dir.glob("*")} + return [p for p in pathlist if Path(p).stem in source_stems] \ No newline at end of file From fd675f5701840018a34fc4a5bf939f73e333e3ca Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 10:49:35 -0500 Subject: [PATCH 146/222] shift break to continue so valid values continue processing --- lib/datura/python/omeka.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 01009f6f4..dd3d874fc 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -58,7 +58,7 @@ def prepare_item_payload_using_template(ctx, terms, template_id): "Data type %r for term %r not allowed by template; skipping value", value['type'], term ) - break + continue if 'type' not in value: # Infer a data type from the template definition. @@ -71,7 +71,7 @@ def prepare_item_payload_using_template(ctx, terms, template_id): else: # Cannot determine a type; skip this value. logger.warning("Cannot determine data type for term %r; skipping value",term) - break + continue if "property_id" in value: # Value was already formatted by a prior call; append as-is to From 77e5ec574e4b452ca0b5fe51523cd0d0c791c144 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 11:01:24 -0500 Subject: [PATCH 147/222] clear out redundant check for already_linked values --- lib/datura/python/api_fields.py | 35 +++++++++++++-------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index dfec7fdc4..2222a6067 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -394,26 +394,19 @@ def link_item_record(ctx, item, key, values, item_set=False, filter_property="dc resource_type = "resource:itemset" if item_set else "resource:item" for omeka_id in omeka_ids: - # Guard against duplicate links — check whether this Omeka ID is - # already present in the list before appending. - already_linked = ( - item[key] and - omeka_id in [v.get("value_resource_id") for v in item[key]] - ) - if not already_linked: - prop_value = { - "type": resource_type, - "value": omeka_id, - } - formatted = ctx.client.prepare_property_value(prop_value, prop_id) - - if item_set: - # The item-sets plugin requires these extra fields in addition - # to what prepare_property_value generates. - formatted['@id'] = '{}/item_sets/{}'.format(ctx.client.api_url, omeka_id) - formatted['value_resource_id'] = omeka_id - formatted['value_resource_name'] = 'item_sets' - - item[key].append(formatted) + prop_value = { + "type": resource_type, + "value": omeka_id, + } + formatted = ctx.client.prepare_property_value(prop_value, prop_id) + + if item_set: + # The item-sets plugin requires these extra fields in addition + # to what prepare_property_value generates. + formatted['@id'] = '{}/item_sets/{}'.format(ctx.client.api_url, omeka_id) + formatted['value_resource_id'] = omeka_id + formatted['value_resource_name'] = 'item_sets' + + item[key].append(formatted) return item From 10cacab243e9c297f85cdfd738f186b7fee771fd Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 11:06:13 -0500 Subject: [PATCH 148/222] check for all required keys at once and give user comprehensive feedback about any missing keys --- lib/datura/python/omeka_context.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index f12956afc..0b3b46d0d 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -355,8 +355,7 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, """ # ---- Validate required config keys -------------------------------- # Validate up front so that failures are immediate and descriptive. - # A missing key surfaced here gives a clear error message; the same - # key missing inside a loop gives an opaque KeyError mid-run. + # Checks for all keys so user is alerted to any missing key at the outset. required_keys = [ "omeka_server", "key_identity", @@ -364,14 +363,14 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, "resource_template", "omeka_data_base", ] - for key in required_keys: - if key not in env_config: - raise OmekaConfigError(RED + - "Missing required config key {!r}. " - "Check the 'default' or {!r} section of config/private.yml." - .format(key, environment) - + RESET - ) + missing_keys = [key for key in required_keys if key not in env_config] + if missing_keys: + raise OmekaConfigError(RED + + "Missing required config key(s): {}. " + "Check the 'default' or {!r} section of config/private.yml." + .format(missing_keys, environment) + + RESET + ) # ---- Validate environment-specific item_set --------------------------- if "item_set" not in env_config: From 3efbe10acbcc7b6a317c1419830cfced873f263d Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 11:55:07 -0500 Subject: [PATCH 149/222] add rest-client to file with parent FileType class (rather than files with subclasses) --- lib/datura/file_type.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index c7e6b7914..f6523b907 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -1,5 +1,6 @@ require "json" require "open3" +require "rest-client" class FileType From 6d41fa86777c2a215b09c9885a382a1eb08e3dff Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 12:33:59 -0500 Subject: [PATCH 150/222] move error reporting into omeka_context --- lib/datura/helpers.rb | 17 ++--------------- lib/datura/python/html_and_media_ingest.py | 6 ------ lib/datura/python/json_to_omeka.py | 6 ------ lib/datura/python/omeka_context.py | 8 +++----- 4 files changed, 5 insertions(+), 32 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 9ea0eb88c..1e86e9f2d 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -2,7 +2,6 @@ require 'net/http' require 'nokogiri' require 'shellwords' -require 'tempfile' require 'yaml' module Datura::Helpers @@ -144,10 +143,9 @@ def self.construct_auth_header(options) def self.run_omeka_script(script_path, options) ''' - Build and run a Python Omeka posting script, then print an error summary. + Build and run a Python Omeka posting script. - Handles tempfile creation for the error count handoff, common CLI flag - forwarding (-e, -r, -m), and the "N Omeka posting error(s)" output line. + Handles common CLI flag forwarding (-e, -r, -m). Called by bin/post_omeka and bin/post_omeka_html. Parameters: @@ -158,24 +156,13 @@ def self.run_omeka_script(script_path, options) puts "Omeka script not found at #{script_path}".red return end - error_file = Tempfile.new(["omeka_errors", ".txt"]) - error_file_path = error_file.path - error_file.close command = ["python3", script_path] command.append("-e", Shellwords.escape(options["environment"])) if options["environment"] command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] command.append("-f", Shellwords.escape(options["format"])) if options["format"] command.append("-m") if options["media_skip"] command.append("-j") if options["json_output"] - command.append("--error-file", error_file_path) system(*command) - omeka_errors = begin - Integer(File.read(error_file_path).strip) - rescue - 0 - end - File.unlink(error_file_path) rescue nil - puts "#{omeka_errors} Omeka posting error(s)" end end diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index b006390b2..79d1669f6 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -119,12 +119,6 @@ def _parse_args(): dest="log_level", help="Set the logging verbosity (default: INFO).", ) - parser.add_argument( - "--error-file", - dest="error_file", - default=None, - help="If provided, write the integer error count to this file before exiting.", - ) return parser.parse_args() diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index c70470f18..226c48241 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -126,12 +126,6 @@ def _parse_args(): dest="log_level", help="Set the logging verbosity (default: INFO).", ) - parser.add_argument( - "--error-file", - dest="error_file", - default=None, - help="If provided, write the integer error count to this file before exiting.", - ) return parser.parse_args() diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 0b3b46d0d..5a61ece9a 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -584,20 +584,18 @@ def report_errors(self): def finish_run(ctx, args, start_time): """ - Report errors, write count to --error-file if provided, print timing, and exit. + Report errors, print error count and timing, and exit. Called at the end of each Omeka entrypoint script's main() function. Exits 0 on success, 1 if any errors were recorded. Parameters: * ctx - OmekaContext whose _errors list is inspected - * args - argparse.Namespace; checked for optional error_file attribute + * args - argparse.Namespace (unused; kept for call-site compatibility) * start_time - float from time.time() captured at the top of main() """ ctx.report_errors() - if getattr(args, "error_file", None): - with open(args.error_file, "w") as f: - f.write(str(len(ctx._errors))) + print(f"{len(ctx._errors)} Omeka posting error(s)") elapsed = int(time.time() - start_time) hours, rem = divmod(elapsed, 3600) mins, secs = divmod(rem, 60) From ce6256de4ef205a17de20698c2fa178a0b071156 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 10 Jun 2026 17:00:00 -0500 Subject: [PATCH 151/222] make title required, alert user if it is missing and skip item --- lib/datura/python/html_and_media_ingest.py | 5 +++++ lib/datura/python/json_to_omeka.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 79d1669f6..8c7a466a3 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -377,6 +377,11 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): logger.warning("Skipping item without identifier in %s", filename) continue + title = json_item.get("title") + if not title: + logger.warning("Skipping item without title in %s", filename) + continue + # --- Look up the item in Omeka --- try: matching_items = ctx.client.filter_items_by_property( diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 226c48241..302171c77 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -171,6 +171,11 @@ def post_items(ctx, pathlist, json_output_dir=None): logger.warning("Skipping item without identifier in %s", filename) continue + title = json_item.get("title") + if not title: + logger.warning("Skipping item without title in %s", filename) + continue + if json_output_dir is not None: # JSON output mode: build the payload and write it to disk # rather than to the Omeka API. @@ -339,6 +344,11 @@ def link_items(ctx, pathlist): logger.debug("Skipping item without identifier in %s", filename) continue + title = json_item.get("title") + if not title: + logger.warning("Skipping item without title in %s", filename) + continue + try: matching_items = ctx.client.filter_items_by_property( filter_property="dcterms:identifier", From bd9cc9ba7a9740508e7d70634ddbac2bc58fd305 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 09:39:24 -0500 Subject: [PATCH 152/222] add 403 to 401 auth error handling --- lib/datura/python/html_and_media_ingest.py | 4 ++-- lib/datura/python/omeka_context.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 8c7a466a3..4034b620b 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -149,9 +149,9 @@ def delete_media_items(ctx, matching_item): logger.info("Deleting media item %s", media_id) ctx.client.delete_resource(media_id, "media") except HTTPError as err: - if err.response.status_code == 401: + if err.response.status_code == 401 or err.response.status_code == 403: raise OmekaAuthError( - "Omeka S returned 401 Unauthorized. " + "Omeka S returned 401 Unauthorized or 403 Forbidden. " "Check that key_identity and key_credential in config/private.yml are correct. " "You may also need to be logged onto the VPN." ) from err diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 5a61ece9a..0dc2c6a82 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -104,7 +104,7 @@ class OmekaConfigError(OmekaError): class OmekaAuthError(OmekaConfigError): """ - Raised when the Omeka S API returns 401 Unauthorized. + Raised when the Omeka S API returns 401 Unauthorized or 403 Forbidden. This is always fatal: if credentials are wrong every API call will fail, so the process exits immediately rather than accumulating per-item failures. @@ -116,12 +116,12 @@ class OmekaAuthError(OmekaConfigError): def _is_unauthorized(err): - """Return True if err is an HTTP 401 response error from the requests library.""" + """Return True if err is an HTTP 401 or 403 response error from the requests library.""" try: from requests.exceptions import HTTPError return isinstance(err, HTTPError) and getattr( getattr(err, "response", None), "status_code", None - ) == 401 + ) in {401, 403} except ImportError: return False @@ -556,7 +556,7 @@ def record_error(self, err): cause = getattr(err, "cause", None) if _is_unauthorized(cause): raise OmekaAuthError(RED + - "Omeka S returned 401 Unauthorized. " + "Omeka S returned 401 Unauthorized or 403 Forbidden. " "Check that key_identity and key_credential in config/private.yml are correct. " "You may also need to be logged onto the VPN." + RESET From c43a13dec89c9f3291cd2cc856ce4fc4d9757fbd Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 09:51:12 -0500 Subject: [PATCH 153/222] remove unused error class definitions --- lib/datura/python/html_and_media_ingest.py | 2 -- lib/datura/python/json_to_omeka.py | 2 -- lib/datura/python/omeka_context.py | 22 +--------------------- 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 4034b620b..67cf8f42a 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -43,8 +43,6 @@ OmekaConfigError, OmekaContext, OmekaMediaError, - OmekaMultipleMatchesError, - OmekaItemNotFoundError, configure_logging, finish_run, ) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 302171c77..151ab6a54 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -43,8 +43,6 @@ OmekaAPIError, OmekaConfigError, OmekaContext, - OmekaItemNotFoundError, - OmekaMultipleMatchesError, configure_logging, finish_run, ) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 0dc2c6a82..36e35374c 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -147,27 +147,7 @@ def __init__(self, identifier, operation, cause): super().__init__( "{} failed for {!r}: {}".format(operation, identifier, cause) ) - - -class OmekaItemNotFoundError(OmekaError): - """ - Raised when an item lookup returns zero results but exactly one was expected. - - Typical causes: - - An item was not ingested during the posting pass before the linking pass ran - - An identifier was changed between runs, leaving the old Omeka record orphaned - """ - - -class OmekaMultipleMatchesError(OmekaError): - """ - Raised when an item lookup returns more than one result for a given identifier. - - Identifiers should be unique within an item set. Multiple matches indicate a - data integrity problem that must be resolved in the Omeka admin UI before the - affected item can be updated automatically. - """ - + class OmekaMediaError(OmekaError): """ From 464a8a64d8edb26dc2865489199582bd47305cb6 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 10:23:31 -0500 Subject: [PATCH 154/222] add error message for ModuleNotFoundError --- lib/datura/python/html_and_media_ingest.py | 38 ++++++++++++++-------- lib/datura/python/json_to_omeka.py | 33 +++++++++++++------ 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 67cf8f42a..62bd1e65f 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -32,20 +32,30 @@ import time from pathlib import Path -import requests -from requests.exceptions import HTTPError - -import omeka -from omeka import filter_items, filter_items_by_date, filter_items_by_format -from omeka_context import ( - OmekaAPIError, - OmekaAuthError, - OmekaConfigError, - OmekaContext, - OmekaMediaError, - configure_logging, - finish_run, -) +try: + import requests + from requests.exceptions import HTTPError + + import omeka + from omeka import filter_items, filter_items_by_date, filter_items_by_format + from omeka_context import ( + OmekaAPIError, + OmekaAuthError, + OmekaConfigError, + OmekaContext, + OmekaMediaError, + configure_logging, + finish_run, + ) +except ModuleNotFoundError as err: + raise SystemExit( + "\033[31m" + "ERROR: {}\n" + "A required Python package could not be found. " + "You may need to ensure the virtual environment is activated before running this script.\n" + "You may also need to be connected to the VPN." + "\033[0m".format(err) + ) from err # Module-level logger. Records from this module appear as # "html_and_media_ingest" in log output. diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 151ab6a54..bb3301bfe 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -37,16 +37,29 @@ import time from pathlib import Path -import api_fields -import omeka -from omeka_context import ( - OmekaAPIError, - OmekaConfigError, - OmekaContext, - configure_logging, - finish_run, -) -from omeka import filter_items, filter_items_by_date, filter_items_by_format, prepare_item_payload_using_template +class OmekaSetupError(Exception): + """Raised when a required import is unavailable, e.g. venv not activated.""" + +try: + import api_fields + import omeka + from omeka_context import ( + OmekaAPIError, + OmekaConfigError, + OmekaContext, + configure_logging, + finish_run, + ) + from omeka import filter_items, filter_items_by_date, filter_items_by_format, prepare_item_payload_using_template +except ModuleNotFoundError as err: + raise SystemExit( + "\033[31m" + "ERROR: {}\n" + "A required Python package could not be found. " + "You may need to ensure the virtual environment is activated before running this script.\n" + "You may also need to be connected to the VPN." + "\033[0m".format(err) + ) from err # Module-level logger. Records from this module appear as "json_to_omeka" # in log output so they can be filtered independently from other modules. From 19fca49125faa6e8b14c3df083b5505edab10e5d Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 10:33:03 -0500 Subject: [PATCH 155/222] remove extraneous media-skip note --- lib/datura/python/json_to_omeka.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index bb3301bfe..4ea1d88af 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -79,10 +79,6 @@ def _parse_args(): * regex - optional file-filter pattern string, or None * log_level - logging level string, default "INFO" - Note: this entrypoint has no --media-skip flag. That flag belongs only - to html_and_media_ingest.py, which handles media re-ingestion. - getattr(args, "media_skip", False) in OmekaContext.from_args() handles - its absence gracefully. """ parser = argparse.ArgumentParser( description="Post Datura ES JSON output to an Omeka S instance." From 92bcb551fdeb77e985555446bfe79c14768623a5 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 11:17:19 -0500 Subject: [PATCH 156/222] cleanup --- bin/post_omeka | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/post_omeka b/bin/post_omeka index ee8e16edb..082ddd0ba 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -52,7 +52,6 @@ ARGV.delete("--json-output") ARGV.unshift("-x", "es", "-o", "-t") #create DataManager before conditional run manager = Datura::DataManager.new -#skip generation step with option -s if generate_es manager.run end From 46d5c060410447fa8e43cde5254fc8777b4f8bc8 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 13:10:07 -0500 Subject: [PATCH 157/222] move date pattern into overrides --- lib/datura/python/field_definitions.py | 13 +++---------- lib/datura/python/omeka_overrides_example.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index cbf421d8b..fe4cdc073 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -52,9 +52,7 @@ def uriData(self, json): uri_data = json.get("uri_data", None) if uri_data: # Strip the original path and reconstruct the URI under the - # collection's configured media base URL. The base URL comes - # from the constructor rather than a global so this class can - # be instantiated safely in tests without a live config file. + # collection's configured media base URL. filename = uri_data.split("/")[-1] new_uri_data = "{}/{}".format(self._omeka_data_base, filename) return new_uri_data @@ -71,16 +69,11 @@ def contributor(self, json): contributor_names = [contributor['name'] for contributor in json.get("contributor") or [] if 'name' in contributor] return contributor_names + # NOTE: use Pattern 5 in omeka_overrides if automatic conversion of dates with year or month only to yyyy-01-01 + # back to yyyy for Omeka is desired def date(self, json): date_to_parse = json.get("date", None) return date_to_parse - # Uncomment the below and remove the above line if this fix is needed - # if date_to_parse and "-01-01" in date_to_parse: - # #dates are automatically converted by Datura to yyyy-01-01 if month and date are missing - # #convert such dates back to yyyy for Omeka S (since it is allowed by the date parser) - # return datetime.strptime(date_to_parse, "%Y-%m-%d").year - # else: - # return date_to_parse def dateYear(self, json): date_to_parse = json.get("date", None) diff --git a/lib/datura/python/omeka_overrides_example.py b/lib/datura/python/omeka_overrides_example.py index 8020c2ca2..7d98abf72 100644 --- a/lib/datura/python/omeka_overrides_example.py +++ b/lib/datura/python/omeka_overrides_example.py @@ -27,4 +27,14 @@ class CustomFields(FieldDefinitions): # Pattern 4: transform a value (e.g. reformat a date or strip whitespace) # def dateDisplay(self, json): # raw = json.get("date_display", None) - # return raw.strip() if raw else None \ No newline at end of file + # return raw.strip() if raw else None + + # Pattern 5: date display fix for -01-01 + # dates are automatically converted by Datura to yyyy-01-01 if month and date are missing + # use the below to convert such dates back to yyyy for Omeka S (since it is allowed by the date parser) + # def date(self, json): + # date_to_parse = json.get("date", None) + # if date_to_parse and "-01-01" in date_to_parse: + # return datetime.strptime(date_to_parse, "%Y-%m-%d").year + # else: + # return date_to_parse \ No newline at end of file From 49754760e4cdeee64728ec8db5cc747ed258fd73 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 13:10:23 -0500 Subject: [PATCH 158/222] cleanup --- lib/datura/python/api_fields.py | 4 +-- lib/datura/python/html_and_media_ingest.py | 19 +++++------- lib/datura/python/json_to_omeka.py | 35 ++++++++++------------ lib/datura/python/omeka.py | 4 +-- lib/datura/python/omeka_context.py | 8 ++--- 5 files changed, 30 insertions(+), 40 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 2222a6067..dd502b798 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -49,7 +49,7 @@ def build_item_dict(ctx, json_item, existing_item): try: # Load the collection-specific field definitions once during OmekaContext # initialization (collection-specific CustomFields subclass if omeka_overrides.py - # is present in scripts/python, otherwise the defaultFieldDefinitions). + # is present in scripts/python, otherwise the default FieldDefinitions). fields = ctx.fields # Start from the existing Omeka item dict when updating, or an empty @@ -204,7 +204,6 @@ def prepare_item(ctx, row, existing_item=None): Returns the built item dict, or raises ValueError if field extraction fails. """ - # TODO: add conditional logic here for items that need a different template return build_item_dict(ctx, row, existing_item) @@ -222,7 +221,6 @@ def link_records(ctx, row, existing_item): Returns the updated item dict. """ - # TODO: add conditional logic here if different relationship schemas are needed return link_item(ctx, row, existing_item) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 62bd1e65f..3f517c083 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -20,7 +20,7 @@ python3 html_and_media_ingest.py -m # skip items that already have media python3 html_and_media_ingest.py --log-level DEBUG -The script is invoked by bin/post_omeka_html in the Datura gem. The Ruby +The script is invoked by bin/post_omeka_html in the Datura gem. The Ruby wrapper passes -e, -r, and -m arguments from its own CLI. """ @@ -57,7 +57,7 @@ "\033[0m".format(err) ) from err -# Module-level logger. Records from this module appear as +# Module-level logger. Records from this module appear as # "html_and_media_ingest" in log output. logger = logging.getLogger(__name__) @@ -274,7 +274,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): ctx.client.add_media_to_item(matching_item["o:id"], thumbnail_local, payload=media_payload) except FileNotFoundError: # The download step wrote the file, but something removed it between - # download and upload. Unlikely in practice but handled explicitly + # download and upload. Unlikely in practice but handled explicitly # so the error message is clear. logger.warning( "Thumbnail file %s not found at upload time; skipping", @@ -294,15 +294,12 @@ def ingest_html(ctx, json_item, matching_item, html_dir): The HTML ingester reads content from payload["data"]["html"] rather than from the uploaded file bytes; the file is opened only to read its content - into memory. The Omeka S "html" ingester stores the markup directly in + into memory. The Omeka S "html" ingester stores the markup directly in the database, making it searchable and renderable within Omeka. Skips silently if: * The .html file does not exist at html_dir/.html. * The file exists but is empty or contains only whitespace. - (An empty HTML file would create a blank media object in Omeka; this - guard prevents that. The root cause — an XSLT transform producing empty - output — should be investigated in the Datura XSLT/transform layer.) Parameters: * ctx - OmekaContext @@ -318,14 +315,12 @@ def ingest_html(ctx, json_item, matching_item, html_dir): html_content = file.read() except FileNotFoundError: # A missing HTML file is common for items that have no text - # representation (e.g. pure image records). Log at INFO so operators + # representation (e.g. pure image records). Log at INFO so users # can see which items were skipped without it being alarming. logger.info("HTML file %s not found; skipping", file_path) return - # Guard against empty or whitespace-only files. An empty POST would - # create a blank HTML media object in Omeka, which is both incorrect and - # misleading when viewing the item in the admin UI. + # Guard against empty or whitespace-only files. if not html_content.strip(): logger.warning( "HTML file for %r is empty; skipping. " @@ -435,7 +430,7 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): continue # --- Media pipeline --- - # Delete first, then re-upload. Order matters: thumbnail must be + # Delete first, then re-upload. Order matters: thumbnail must be # uploaded before HTML so that Omeka designates the image as # primary_media. delete_media_items(ctx, matching_item) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 4ea1d88af..46a5bc1cc 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -1,7 +1,7 @@ """ json_to_omeka.py -Entrypoint script: reads Datura-generated Elasticsearch JSON files and posts +Entrypoint script: reads Datura-generated ElasticSearch JSON files and posts each item to an Omeka S instance. The script runs in two sequential passes: @@ -37,9 +37,6 @@ import time from pathlib import Path -class OmekaSetupError(Exception): - """Raised when a required import is unavailable, e.g. venv not activated.""" - try: import api_fields import omeka @@ -101,9 +98,9 @@ def _parse_args(): default=False, help=( "Write Omeka S item payloads to output//omeka/ instead " - "of posting to the API. No items are created or updated in Omeka. " + "of posting to the API. No items are created or updated in Omeka. " "An API connection is still required for property ID lookups and " - "template validation. The link pass is skipped because no live " + "template validation. The link pass is skipped because no live " "Omeka item IDs are available." ), ) @@ -112,7 +109,7 @@ def _parse_args(): default=None, help=( "Optional regex pattern to restrict processing to matching " - "file paths. Example: -r 'abc123' processes only files whose " + "file paths. Example: -r 'abc123' processes only files whose " "path contains 'abc123'." ), ) @@ -122,7 +119,7 @@ def _parse_args(): dest="update_time", help=( "Only process items whose source file was modified at or after " - "this date/time. Accepts 'today', a date (2015-01-01), or " + "this date/time. Accepts 'today', a date (2015-01-01), or " "date-time (2015-01-01T18:24)." ), ) @@ -144,7 +141,7 @@ def post_items(ctx, pathlist, json_output_dir=None): """ First pass: create or update Omeka items for every JSON record. - Iterates over all JSON files in pathlist. For each record: + Iterates over all JSON files in pathlist. For each record: * Items whose identifier already exists in Omeka are updated in place. * Items not yet in Omeka are created from scratch. * Items that return multiple Omeka matches are skipped with a warning @@ -153,7 +150,7 @@ def post_items(ctx, pathlist, json_output_dir=None): * Items whose identifier field is falsy are skipped with a warning. Per-item errors (API failures, malformed payload) are recorded via - ctx.record_error() and do NOT halt the run. Fatal errors (wrong + ctx.record_error() and do NOT halt the run. Fatal errors (wrong credentials, missing config) raise exceptions that propagate to main(). Parameters: @@ -239,7 +236,7 @@ def add_new_item(ctx, json_item, template_number): Build the Omeka payload for a new item and POST it to the API. Calls api_fields.prepare_item() to extract and format each field from - the Datura JSON record. If preparation produces no payload (e.g. all + the Datura JSON record. If preparation produces no payload (e.g. all fields were absent), the item is skipped with a warning rather than POSTing an empty object. @@ -255,7 +252,7 @@ def add_new_item(ctx, json_item, template_number): logger.warning("Could not prepare payload for %r; skipping", identifier) return - # Log the identifier we are about to create. Use .get() with a default + # Log the identifier we are about to create. Use .get() with a default # rather than direct dict access so a missing dcterms:identifier key # does not raise KeyError and halt the run. title_val = ( @@ -324,12 +321,12 @@ def link_items(ctx, pathlist): """ Second pass: populate relational fields between Omeka items. - Re-reads the same JSON files processed in pass 1. For each record, + Re-reads the same JSON files processed in pass 1. For each record, resolves the Omeka item IDs of related items (has_part, has_source, etc.) and PATCHes the item with link values. A separate pass is necessary because linked items must already exist in - Omeka before they can be referenced. Running this pass after all items + Omeka before they can be referenced. Running this pass after all items have been created (or updated) in pass 1 guarantees that the target items are present. @@ -442,7 +439,7 @@ def main(): 2. Configure the root logger (before any other work so all output is captured at the right level). 3. Build OmekaContext — loads config/private.yml, validates required keys, - initialises the authenticated API client. Exits with a descriptive + initialises the authenticated API client. Exits with a descriptive error message if the config is missing or malformed (OmekaConfigError). 4. Discover JSON files under output//es/. 5. Apply regex filter if -r was passed. @@ -461,8 +458,8 @@ def main(): # OmekaContext.from_args() raises OmekaConfigError (a subclass of # OmekaError) if config/private.yml is missing, unparseable, or missing - # a required key. Let this propagate to the top level — configuration - # errors are fatal and should produce a clear traceback for the operator. + # a required key. Let this propagate to the top level — configuration + # errors are fatal and should produce a clear traceback. ctx = OmekaContext.from_args(args) # Resolve the ES output directory for the requested environment. @@ -503,9 +500,9 @@ def main(): logger.info("Starting pass 1: item posting") post_items(ctx, pathlist) - # Reset the API client between passes. ctx.reset_client() re-instantiates + # Reset the API client between passes. ctx.reset_client() re-instantiates # OmekaAPIClient with the same credentials, giving a fresh connection for - # the second round of requests. The property ID cache is preserved because + # the second round of requests. The property ID cache is preserved because # term-to-ID mappings do not change between passes. logger.info("Pass 1 complete. Resetting API client for pass 2.") ctx.reset_client() diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index dd3d874fc..50e799674 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -18,7 +18,7 @@ def prepare_item_payload_using_template(ctx, terms, template_id): """ Build an item payload, validating terms and values against a resource template. - Behaviour: + Behavior: - Terms not present in the template are logged and dropped from the payload. - Values whose data type does not match the template definition are dropped. - If no data type is supplied for a value, the template default is used, @@ -163,6 +163,6 @@ def filter_items_by_format(format_type, pathlist): * pathlist - iterable of pathlib.Path or string paths to filter """ source_dir = Path.cwd() / "source" / format_type - # Build the set of stems once instead of globbing per item + # Build the set of stems once source_stems = {sf.stem for sf in source_dir.glob("*")} return [p for p in pathlist if Path(p).stem in source_stems] \ No newline at end of file diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 36e35374c..2d5bb2074 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -32,7 +32,7 @@ CYAN = "\033[36m" GREEN = "\033[32m" RED = "\033[31m" -RED2 = "\033[1;31m" +BRIGHTRED = "\033[1;31m" YELLOW = "\033[33m" RESET = "\033[0m" @@ -42,7 +42,7 @@ class ColoredConsoleHandler(logging.StreamHandler): logging.INFO: GREEN, logging.WARNING: YELLOW, logging.ERROR: RED, - logging.CRITICAL: RED2, + logging.CRITICAL: BRIGHTRED, } RESET = RESET def emit(self, record): @@ -60,9 +60,9 @@ def configure_logging(level="INFO"): "logs/python.log", maxBytes=5 * 1024 * 1024, backupCount=3 ) file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")) - file_handler.setLevel(logging.DEBUG) # always save everything + file_handler.setLevel(logging.DEBUG) # always save everything to logs - # Console handler — colored, respects the requested level + # Console handler — respects the requested level console_handler = ColoredConsoleHandler() console_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-8s %(name)s: %(message)s")) console_handler.setLevel(numeric_level) From 9c19bf8b0300286c75fe119d26173d81da4e125a Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 13:59:36 -0500 Subject: [PATCH 159/222] adjust env_config to fail if no environment config is present --- lib/datura/python/omeka_context.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 2d5bb2074..efa7620ec 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -242,17 +242,8 @@ def from_args(cls, args): # Load the environment-specific section (primarily contains item_set). # If the section is absent (e.g. an unrecognised environment name was - # passed), log a warning and fall back to an empty dict — item_set_id - # will be None and the run will proceed without filtering by item set. - try: - env_config = cls._load_config(conf_path, env=args.environment) - except OmekaConfigError: - logger.warning( - "No config section found for environment %r; " - "item_set will be None and items will not be scoped to a set.", - args.environment, - ) - env_config = {} + # passed), raise an error and exit. + env_config = cls._load_config(conf_path, env=args.environment) # Merge so that environment-specific values override defaults, giving # collections the ability to override any default key (e.g. From 325b0c2238ee211cb51feb0c34513b4401c240d7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 14:06:21 -0500 Subject: [PATCH 160/222] cleanup --- lib/datura/python/omeka_context.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index efa7620ec..ee34cd788 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -420,12 +420,10 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, def item_set_id(self): # type: () -> Optional[int] """ - The Omeka item set ID for the current environment, or None. + The Omeka item set ID for the current environment. Stored in the environment-specific config section so that development - and production ingests target different item sets. Returns None if no - item_set key is present (e.g. running locally without a complete - private.yml, or using an environment that has no item_set configured). + and production ingests can target different item sets. """ return self._env_config.get("item_set") From e279f7b5dcb44ca46b87874d1f79b26ebd3a0977 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 14:15:19 -0500 Subject: [PATCH 161/222] shorten path in error message --- lib/datura/python/html_and_media_ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 3f517c083..124dc1466 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -482,7 +482,7 @@ def main(): logger.info( "Found %d JSON file(s) in %s (environment=%r, media_skip=%s)", len(pathlist), - json_dir, + "output/{}/es".format(ctx.environment), ctx.environment, ctx.media_skip, ) From d105e6834f17ac2d53a071ac3c731d400209e10b Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 11 Jun 2026 14:30:48 -0500 Subject: [PATCH 162/222] use relative paths in missing title and identifier error messages --- lib/datura/python/html_and_media_ingest.py | 5 +++-- lib/datura/python/json_to_omeka.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 124dc1466..2f4d31f8e 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -371,18 +371,19 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): """ for path in pathlist: filename = str(path) + rel = path.relative_to(Path.cwd()) with open(filename) as jsonfile: json_items = json.load(jsonfile) for json_item in json_items: identifier = json_item.get("identifier") if not identifier: - logger.warning("Skipping item without identifier in %s", filename) + logger.warning("Skipping item without identifier in %s", rel) continue title = json_item.get("title") if not title: - logger.warning("Skipping item without title in %s", filename) + logger.warning("Skipping item without title in %s", rel) continue # --- Look up the item in Omeka --- diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 46a5bc1cc..2bbcef0f6 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -161,6 +161,7 @@ def post_items(ctx, pathlist, json_output_dir=None): """ for path in pathlist: filename = str(path) + rel = path.relative_to(Path.cwd()) with open(filename) as jsonfile: json_items = json.load(jsonfile) @@ -172,12 +173,12 @@ def post_items(ctx, pathlist, json_output_dir=None): identifier = json_item.get("identifier") if not identifier: # Records without an identifier cannot be matched or created. - logger.warning("Skipping item without identifier in %s", filename) + logger.warning("Skipping item without identifier in %s", rel) continue title = json_item.get("title") if not title: - logger.warning("Skipping item without title in %s", filename) + logger.warning("Skipping item without title in %s", rel) continue if json_output_dir is not None: @@ -339,18 +340,19 @@ def link_items(ctx, pathlist): """ for path in pathlist: filename = str(path) + rel = path.relative_to(Path.cwd()) with open(filename) as jsonfile: json_items = json.load(jsonfile) for json_item in json_items: identifier = json_item.get("identifier") if not identifier: - logger.debug("Skipping item without identifier in %s", filename) + logger.debug("Skipping item without identifier in %s", rel) continue title = json_item.get("title") if not title: - logger.warning("Skipping item without title in %s", filename) + logger.warning("Skipping item without title in %s", rel) continue try: From 21e479f53c3e38cb0f6d5da4675579c5340020aa Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 15 Jun 2026 15:59:31 -0500 Subject: [PATCH 163/222] move from deprecated rest-client gem to built-in net/http gem --- datura.gemspec | 1 - lib/datura/elasticsearch/alias.rb | 31 ++++++----- lib/datura/elasticsearch/index.rb | 92 +++++++++++++++---------------- lib/datura/file_type.rb | 6 +- lib/datura/helpers.rb | 20 ++++++- 5 files changed, 84 insertions(+), 66 deletions(-) diff --git a/datura.gemspec b/datura.gemspec index f4442aa27..3ca770de5 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -59,7 +59,6 @@ Gem::Specification.new do |spec| spec.required_ruby_version = "~> 3.1" spec.add_runtime_dependency "colorize", "~> 1.1" spec.add_runtime_dependency "nokogiri", "~> 1.18" - spec.add_runtime_dependency "rest-client", "~> 2.1" spec.add_runtime_dependency "pdf-reader", "~> 2.15" spec.add_development_dependency "byebug", "~> 11.0" # leaving this constraint as-is to avoid possible conflicts with diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index 4d6a3a118..42acc5430 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -1,5 +1,4 @@ require "json" -require "rest-client" require_relative "./../elasticsearch.rb" @@ -20,15 +19,16 @@ def self.add { add: { alias: ali, index: idx } } ] } - RestClient.post(base_url, data.to_json, @auth_header.merge({ content_type: :json })) { |res, req, result| - if result.code == "200" - puts res - puts "Successfully added alias #{ali}. Current alias list:" - puts list - else - raise "#{result.code} error managing aliases: #{res}" - end - } + response = Datura::Helpers.es_http_request("POST", base_url, + body: data.to_json, + headers: (@auth_header || {}).merge("Content-Type" => "application/json")) + if response.code == "200" + puts response.body + puts "Successfully added alias #{ali}. Current alias list:" + puts list + else + raise "#{response.code} error managing aliases: #{response.body}" + end end def self.delete @@ -40,16 +40,19 @@ def self.delete url = File.join(options["es_path"], idx, "_alias", ali) - res = JSON.parse(RestClient.delete(url, @auth_header)) - puts JSON.pretty_generate(res) + response = Datura::Helpers.es_http_request("DELETE", url, + headers: @auth_header || {}) + puts JSON.pretty_generate(JSON.parse(response.body)) list end def self.list options = Datura::Options.new({}).all - res = RestClient.get(File.join(options["es_path"], "_aliases"), ) - JSON.pretty_generate(JSON.parse(res)) + auth = Datura::Helpers.construct_auth_header(options) + response = Datura::Helpers.es_http_request("GET", File.join(options["es_path"], "_aliases"), + headers: auth) + JSON.pretty_generate(JSON.parse(response.body)) end end diff --git a/lib/datura/elasticsearch/index.rb b/lib/datura/elasticsearch/index.rb index 8010a6385..b4ffe0a44 100644 --- a/lib/datura/elasticsearch/index.rb +++ b/lib/datura/elasticsearch/index.rb @@ -1,5 +1,4 @@ require "json" -require "rest-client" require "yaml" require "base64" @@ -36,42 +35,36 @@ def create json = @requested_schema["settings"].to_json puts "Creating ES index for API version #{@options["api_version"]}: #{@pretty_url}" if json && json != "null" - RestClient.put(@pretty_url, json, @auth_header.merge({ content_type: :json })) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error creating Elasticsearch index: #{res}" - end - } + response = Datura::Helpers.es_http_request("PUT", @pretty_url, + body: json, + headers: @auth_header.merge("Content-Type" => "application/json")) else - RestClient.put(@pretty_url, nil, @auth_header) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error creating Elasticsearch index: #{res}" - end - } + response = Datura::Helpers.es_http_request("PUT", @pretty_url, + headers: @auth_header) + end + if response.code == "200" + puts response.body + else + raise "#{response.code} error creating Elasticsearch index: #{response.body}" end end def delete puts "Deleting #{@options["es_index"]} via url #{@pretty_url}" - RestClient.delete(@pretty_url, @auth_header) { |res, req, result| - if result.code != "200" - raise "#{result.code} error deleting Elasticsearch index: #{res}" - end - } + response = Datura::Helpers.es_http_request("DELETE", @pretty_url, + headers: @auth_header) + raise "#{response.code} error deleting Elasticsearch index: #{response.body}" if response.code != "200" end def get_schema - RestClient.get(@mapping_url, @auth_header) { |res, req, result| - if result.code == "200" - JSON.parse(res) - else - raise "#{result.code} error getting Elasticsearch schema: #{res}" - end - } + response = Datura::Helpers.es_http_request("GET", @mapping_url, + headers: @auth_header) + if response.code == "200" + JSON.parse(response.body) + else + raise "#{response.code} error getting Elasticsearch schema: #{response.body}" + end end def get_schema_mapping @@ -114,13 +107,14 @@ def set_schema json = @requested_schema["mappings"].to_json puts "Setting schema: #{@mapping_url}" - RestClient.put(@mapping_url, json, @auth_header.merge({ content_type: :json })) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error setting Elasticsearch schema: #{res}" - end - } + response = Datura::Helpers.es_http_request("PUT", @mapping_url, + body: json, + headers: @auth_header.merge("Content-Type" => "application/json")) + if response.code == "200" + puts response.body + else + raise "#{response.code} error setting Elasticsearch schema: #{response.body}" + end end # doc: ruby hash corresponding with Elasticsearch document JSON @@ -213,13 +207,14 @@ def self.clear_all(options) url = File.join(options["es_path"], options["es_index"], "_delete_by_query?pretty=true") auth_header = Datura::Helpers.construct_auth_header(options) json = { "query" => { "match_all" => {} } } - RestClient.post(url, json.to_json, auth_header.merge({ content_type: :json })) { |res, req, result| - if result.code == "200" - puts res - else - raise "#{result.code} error when clearing entire index: #{res}" - end - } + response = Datura::Helpers.es_http_request("POST", url, + body: json.to_json, + headers: auth_header.merge("Content-Type" => "application/json")) + if response.code == "200" + puts response.body + else + raise "#{response.code} error when clearing entire index: #{response.body}" + end else puts "You typed '#{confirm}'. This is incorrect, exiting program" exit @@ -233,13 +228,14 @@ def self.clear_index(options) if confirmation data = self.build_clear_data(options) auth_header = Datura::Helpers.construct_auth_header(options) - RestClient.post(url, data.to_json, auth_header.merge({content_type: :json })) { |res, req, result| - if result.code == "200" || result.code == "201" - puts res - else - raise "#{result.code} error when clearing index: #{res}" - end - } + response = Datura::Helpers.es_http_request("POST", url, + body: data.to_json, + headers: auth_header.merge("Content-Type" => "application/json")) + if response.code == "200" || response.code == "201" + puts response.body + else + raise "#{response.code} error when clearing index: #{response.body}" + end else puts "come back anytime!" exit diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index f6523b907..7f960083e 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -1,6 +1,5 @@ require "json" require "open3" -require "rest-client" class FileType @@ -82,7 +81,10 @@ def post_es(es) # NOTE: If you need to do partial updates rather than replacement of doc # you will need to add _update at the end of this URL begin - RestClient.put("#{es.index_url}/_doc/#{id}", doc.to_json, @auth_header.merge({:content_type => :json }) ) + response = Datura::Helpers.es_http_request("PUT", "#{es.index_url}/_doc/#{id}", + body: doc.to_json, + headers: @auth_header.merge("Content-Type" => "application/json")) + raise "#{response.code} error posting to Elasticsearch: #{response.body}" unless response.code.start_with?("2") rescue Errno::ECONNREFUSED, SocketError, Errno::ETIMEDOUT => e error = "Could not connect to ElasticSearch at #{es.index_url}. " \ "Confirm you have specified the correct environment " \ diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 671c1511c..2c2a141e0 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -2,6 +2,7 @@ require 'net/http' require 'nokogiri' require 'yaml' +require 'uri' module Datura::Helpers @@ -145,7 +146,24 @@ def self.construct_auth_header(options) "Credentials will be transmitted in cleartext. Use HTTPS in production." end - { "Authorization" => "Basic #{Base64::encode64("#{username}:#{password}")}" } + { "Authorization" => "Basic #{Base64::strict_encode64("#{username}:#{password}")}" } + end + + def self.es_http_request(method, url, body: nil, headers: {}) + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == "https") + req_class = { + "GET" => Net::HTTP::Get, + "PUT" => Net::HTTP::Put, + "POST" => Net::HTTP::Post, + "DELETE" => Net::HTTP::Delete + }.fetch(method.upcase) + req = request_class.new(uri.request_uri) + headers.each { |k, v| req[k.to_s] = v } + req.body = body if body + + http.request(req) end end From d16f2a2830d727ffea9206574d530dacbf76aaf7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 15 Jun 2026 16:03:18 -0500 Subject: [PATCH 164/222] fix @auth_header call to be local rather than calling module-level variable --- lib/datura/elasticsearch/alias.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/datura/elasticsearch/alias.rb b/lib/datura/elasticsearch/alias.rb index 42acc5430..d28751bee 100644 --- a/lib/datura/elasticsearch/alias.rb +++ b/lib/datura/elasticsearch/alias.rb @@ -19,9 +19,10 @@ def self.add { add: { alias: ali, index: idx } } ] } + auth = Datura::Helpers.construct_auth_header(options) response = Datura::Helpers.es_http_request("POST", base_url, body: data.to_json, - headers: (@auth_header || {}).merge("Content-Type" => "application/json")) + headers: auth.merge("Content-Type" => "application/json")) if response.code == "200" puts response.body puts "Successfully added alias #{ali}. Current alias list:" @@ -40,8 +41,9 @@ def self.delete url = File.join(options["es_path"], idx, "_alias", ali) + auth = Datura::Helpers.construct_auth_header(options) response = Datura::Helpers.es_http_request("DELETE", url, - headers: @auth_header || {}) + headers: auth) puts JSON.pretty_generate(JSON.parse(response.body)) list end From 5243eb68504fe4c96e10eb2bae53dd6d96537446 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 15 Jun 2026 16:08:37 -0500 Subject: [PATCH 165/222] shift bundler constraints up; as dev dependency this should not cause problems --- datura.gemspec | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datura.gemspec b/datura.gemspec index 3ca770de5..bcfc11aeb 100644 --- a/datura.gemspec +++ b/datura.gemspec @@ -61,9 +61,7 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "nokogiri", "~> 1.18" spec.add_runtime_dependency "pdf-reader", "~> 2.15" spec.add_development_dependency "byebug", "~> 11.0" - # leaving this constraint as-is to avoid possible conflicts with - # later versions of bundler requiring Ruby > 3.1 - spec.add_development_dependency "bundler", ">= 1.16.0", "< 3.0" + spec.add_development_dependency "bundler", ">= 2.0", "< 5.0" spec.add_development_dependency "minitest", "~> 5.0" spec.add_development_dependency "rake", "~> 13.0" end From 9ca0e0dd80d3421937b84eb06084037abb4971dc Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 15 Jun 2026 16:15:14 -0500 Subject: [PATCH 166/222] fix typo --- lib/datura/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 2c2a141e0..43bf321df 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -159,7 +159,7 @@ def self.es_http_request(method, url, body: nil, headers: {}) "POST" => Net::HTTP::Post, "DELETE" => Net::HTTP::Delete }.fetch(method.upcase) - req = request_class.new(uri.request_uri) + req = req_class.new(uri.request_uri) headers.each { |k, v| req[k.to_s] = v } req.body = body if body From d45fa057248378011920eabaeb336aa68118f30d Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 16 Jun 2026 13:13:58 -0500 Subject: [PATCH 167/222] fix typo --- docs/3_manage/post_omeka.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/3_manage/post_omeka.md b/docs/3_manage/post_omeka.md index d26907da0..e162f527c 100644 --- a/docs/3_manage/post_omeka.md +++ b/docs/3_manage/post_omeka.md @@ -16,7 +16,7 @@ For information on how to override field definitions, see [Omeka Overrides](../2 ### Notes on debugging -The standard way to debug Python scripts is with `breakpoint()`, equivalent to `byebug` in Ruby. Execution will pause and then you can check the contents of variables and try snippets of code from the prompt. Sometimes putting a debugger within an except clause (especially within a loop) makes it difficult to break out of the script and halt execution even if you type `quit`. `CTRL-C` sometimes works in these cases. If `CTRL-C` also fails to halt the script, try running `os._exit(0)` (this is the reason for `import os` in some of the scripts, and if you get an error that the module is not found, then `import os` should be run from the debugger prompt first). For more details on a particular error, look at the error message which is printed in some of the except clauses, check online documentation to see if this error message has additional methods (depending on the error), or use `traceback.print_exc()` to print out the full stack trace. You can also check the logs at `/logs/python.log`. In the case of errors in the HTTP reponse, it may be necessary to look in the logs on the Omeka site. +The standard way to debug Python scripts is with `breakpoint()`, equivalent to `byebug` in Ruby. Execution will pause and then you can check the contents of variables and try snippets of code from the prompt. Sometimes putting a debugger within an except clause (especially within a loop) makes it difficult to break out of the script and halt execution even if you type `quit`. `CTRL-C` sometimes works in these cases. If `CTRL-C` also fails to halt the script, try running `os._exit(0)` (this is the reason for `import os` in some of the scripts, and if you get an error that the module is not found, then `import os` should be run from the debugger prompt first). For more details on a particular error, look at the error message which is printed in some of the except clauses, check online documentation to see if this error message has additional methods (depending on the error), or use `traceback.print_exc()` to print out the full stack trace. You can also check the logs at `/logs/python.log`. In the case of errors in the HTTP response, it may be necessary to look in the logs on the Omeka site. ### 500 error From 5d2554d111613a690249ef0558433ad395967e32 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 16 Jun 2026 15:25:58 -0500 Subject: [PATCH 168/222] reverse nokogiri and byebug changes --- Gemfile.lock | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a30898257..81690a88a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,8 +27,8 @@ GEM mime-types-data (3.2026.0414) minitest (5.27.0) netrc (0.11.0) - nokogiri (1.18.10-arm64-darwin) - racc (~> 1.4) + nokogiri (1.18.10) + mini_portile2 (~> 2.8.2) nokogiri (1.18.10-x86_64-darwin) racc (~> 1.4) pdf-reader (2.15.1) @@ -49,12 +49,11 @@ GEM bigdecimal (~> 3.1) PLATFORMS - arm64-darwin-24 + ruby x86_64-darwin-20 DEPENDENCIES bundler (>= 2.0, < 5.0) - byebug (~> 11.0) datura! minitest (~> 5.0) rake (~> 13.0) From 7d0024da1171872deb243f1d51d48562acf9c3f2 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 16 Jun 2026 15:28:17 -0500 Subject: [PATCH 169/222] fix overlooked nokogiri change --- Gemfile.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Gemfile.lock b/Gemfile.lock index 81690a88a..94a602d61 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -29,6 +29,7 @@ GEM netrc (0.11.0) nokogiri (1.18.10) mini_portile2 (~> 2.8.2) + racc (~> 1.4) nokogiri (1.18.10-x86_64-darwin) racc (~> 1.4) pdf-reader (2.15.1) From 21f3fa120c3db305dd9b0b39e7dd3f07b36eb69f Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 18 Jun 2026 16:52:45 -0500 Subject: [PATCH 170/222] separate out field mappings into overrideable manifest --- lib/datura/python/field_definitions.py | 79 ++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index fe4cdc073..c4fb902a7 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -29,6 +29,85 @@ def __init__(self, omeka_data_base=""): # Stored as a private attribute and accessed only by uriData(). self._omeka_data_base = omeka_data_base + def field_manifest(self): + """ + Declare the ordered list of Omeka property mappings for this collection. + + Each entry is a (omeka_term, method_name, datatype) triple: + * omeka_term - Omeka S property term string, e.g. "dcterms:title" + * method_name - name of the extractor method on this FieldDefinitions + instance to call for each JSON item + * datatype - Omeka data type string + + prepare_item() in api_fields.py iterates this manifest to build the + item payload, calling getattr(ctx.fields, method_name)(row) for each + entry. Overriding field_manifest in a CustomFields subclass is the + recommended way to add collection-specific Omeka properties without + touching prepare_item() itself. + """ + return [ + ("dcterms:title", "title", "literal"), + ("dcterms:identifier", "identifier", "literal"), + ("dh:collection", "collection", "literal"), + ("dh:category", "category", "literal"), + ("dh:category2", "category2", "literal"), + ("dh:uriData", "uriData", "uri"), + ("dcterms:type", "dcterms_type", "literal"), + ("dcterms:creator", "creator", "literal"), + ("dcterms:contributor", "contributor", "literal"), + ("dcterms:date", "date", "numeric:timestamp"), + ("dh:dateDisplay", "dateDisplay", "literal"), + ("dh:dateYear", "dateYear", "literal"), + ("dcterms:description", "description", "literal"), + ("dcterms:format", "dcterms_format", "literal"), + ("dcterms:relation", "relation", "literal"), + ("dcterms:publisher", "publisher", "literal"), + ("dh:biblID", "biblID", "literal"), + ("tei:biblTitle", "biblTitle", "literal"), + ("tei:biblPubPlace", "biblPubPlace", "literal"), + ("bibo:issue", "issue", "literal"), + ("bibo:pageStart", "pageStart", "literal"), + ("bibo:pageEnd", "pageEnd", "literal"), + ("bibo:section", "section", "literal"), + ("bibo:volume", "volume", "literal"), + ("tei:biblTitleA", "biblTitleA", "literal"), + ("tei:biblTitleM", "biblTitleM", "literal"), + ("tei:biblTitleJ", "biblTitleJ", "literal"), + ("dcterms:rightsHolder", "rightsHolder", "literal"), + ("dcterms:license", "license", "literal"), + ("dcterms:subject", "subject", "literal"), + ("dh:topic", "topic", "literal"), + ("dh:category3", "category3", "literal"), + ("dh:category4", "category4", "literal"), + ("dh:category5", "category5", "literal"), + ("dh:note", "note", "literal"), + ("dcterms:abstract", "abstract", "literal"), + ("dh:keyword", "keyword", "literal"), + ("dh:keyword2", "keyword2", "literal"), + ("dh:keyword3", "keyword3", "literal"), + ("dh:keyword4", "keyword4", "literal"), + ("dh:keyword5", "keyword5", "literal"), + ("dcterms:source", "source", "literal"), + ("dcterms:medium", "medium", "literal"), + ("dcterms:extent", "extent", "literal"), + ("dcterms:language", "language", "literal"), + ("dh:box", "box", "literal"), + ("dh:folder", "folder", "literal"), + ("foaf:name", "name", "literal"), + ("dh:spatial_short_name", "spatial_short_name", "literal"), + ("tei:correspSentName", "correspSentName", "literal"), + ("tei:correspSentPlace", "correspSentPlace", "literal"), + ("tei:correspSentDate", "correspSentDate", "numeric:timestamp"), + ("tei:correspDeliveredName", "correspDeliveredName", "literal"), + ("tei:correspDeliveredPlace", "correspDeliveredPlace","literal"), + ("tei:correspDeliveredDate", "correspDeliveredDate", "numeric:timestamp"), + ("tei:distributor", "distributor", "literal"), + ("tei:authority", "authority", "literal"), + ("tei:biblNote", "biblNote", "literal"), + ("dh:annotationsText", "annotationsText", "literal"), + ("dh:itemText", "itemText", "literal"), + ] + def _get_citation(self, json): """Return the citation sub-dict, or {} if absent or null.""" return json.get("citation") or {} From e70b637d2e0621598b256676b518c15ab4135b36 Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 18 Jun 2026 17:13:01 -0500 Subject: [PATCH 171/222] consolidate empty functions and separate logic for overridability --- lib/datura/python/api_fields.py | 177 +++++---------------- lib/datura/python/html_and_media_ingest.py | 106 ++++++++---- 2 files changed, 117 insertions(+), 166 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index dd502b798..900f64737 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -27,18 +27,19 @@ logger = logging.getLogger(__name__) -def build_item_dict(ctx, json_item, existing_item): +def prepare_item(ctx, row, existing_item=None): """ - Map a Datura JSON item to an Omeka S item dict, populating all configured - property fields. + Build a complete Omeka item dict from a Datura JSON record. - Iterates over the ~70 field definitions in FieldDefinitions (or a - collection-specific CustomFields subclass), extracts each value from the - JSON item, and calls update_item_value() to format and attach it. + Iterates over the field manifest declared on ctx.fields, where each entry + specifies an Omeka property term, the extractor method name to call on + ctx.fields, and the Omeka datatype. The manifest is defined by + FieldDefinitions.field_manifest and may be extended by a collection's + CustomFields subclass to add collection-specific properties. Parameters: * ctx - OmekaContext providing config and the property ID cache - * json_item - dict representing one record from the Datura ES output + * row - raw JSON item dict from the Datura ES output * existing_item - existing Omeka item dict to update in-place, or None when creating a new item (an empty dict is used instead) @@ -47,88 +48,17 @@ def build_item_dict(ctx, json_item, existing_item): unexpected way. """ try: - # Load the collection-specific field definitions once during OmekaContext - # initialization (collection-specific CustomFields subclass if omeka_overrides.py - # is present in scripts/python, otherwise the default FieldDefinitions). - fields = ctx.fields - - # Start from the existing Omeka item dict when updating, or an empty - # dict when creating. update_item_value() clears each key before - # writing, so stale values from the existing item are replaced. built_item = existing_item if existing_item else {} - - update_item_value(ctx, built_item, "dcterms:title", fields.title(json_item)) - update_item_value(ctx, built_item, "dcterms:identifier", fields.identifier(json_item)) - update_item_value(ctx, built_item, "dh:collection", fields.collection(json_item)) - update_item_value(ctx, built_item, "dh:category", fields.category(json_item)) - update_item_value(ctx, built_item, "dh:category2", fields.category2(json_item)) - update_item_value(ctx, built_item, "dh:uriData", fields.uriData(json_item), "uri") - update_item_value(ctx, built_item, "dcterms:type", fields.dcterms_type(json_item)) - update_item_value(ctx, built_item, "dcterms:creator", fields.creator(json_item)) - update_item_value(ctx, built_item, "dcterms:contributor", fields.contributor(json_item)) - update_item_value(ctx, built_item, "dcterms:date", fields.date(json_item), "numeric:timestamp") - update_item_value(ctx, built_item, "dh:dateDisplay", fields.dateDisplay(json_item)) - update_item_value(ctx, built_item, "dh:dateYear", fields.dateYear(json_item)) - update_item_value(ctx, built_item, "dcterms:description", fields.description(json_item)) - update_item_value(ctx, built_item, "dcterms:format", fields.dcterms_format(json_item)) - update_item_value(ctx, built_item, "dcterms:relation", fields.relation(json_item)) - update_item_value(ctx, built_item, "dcterms:publisher", fields.publisher(json_item)) - update_item_value(ctx, built_item, "dh:biblID", fields.biblID(json_item)) - update_item_value(ctx, built_item, "tei:biblTitle", fields.biblTitle(json_item)) - update_item_value(ctx, built_item, "tei:biblPubPlace", fields.biblPubPlace(json_item)) - update_item_value(ctx, built_item, "bibo:issue", fields.issue(json_item)) - update_item_value(ctx, built_item, "bibo:pageStart", fields.pageStart(json_item)) - update_item_value(ctx, built_item, "bibo:pageEnd", fields.pageEnd(json_item)) - update_item_value(ctx, built_item, "bibo:section", fields.section(json_item)) - update_item_value(ctx, built_item, "bibo:volume", fields.volume(json_item)) - update_item_value(ctx, built_item, "tei:biblTitleA", fields.biblTitleA(json_item)) - update_item_value(ctx, built_item, "tei:biblTitleM", fields.biblTitleM(json_item)) - update_item_value(ctx, built_item, "tei:biblTitleJ", fields.biblTitleJ(json_item)) - update_item_value(ctx, built_item, "dcterms:rightsHolder", fields.rightsHolder(json_item)) - update_item_value(ctx, built_item, "dcterms:license", fields.license(json_item)) - update_item_value(ctx, built_item, "dcterms:subject", fields.subject(json_item)) - update_item_value(ctx, built_item, "dh:topic", fields.topic(json_item)) - update_item_value(ctx, built_item, "dh:category3", fields.category3(json_item)) - update_item_value(ctx, built_item, "dh:category4", fields.category4(json_item)) - update_item_value(ctx, built_item, "dh:category5", fields.category5(json_item)) - update_item_value(ctx, built_item, "dh:note", fields.note(json_item)) - update_item_value(ctx, built_item, "dcterms:abstract", fields.abstract(json_item)) - update_item_value(ctx, built_item, "dh:keyword", fields.keyword(json_item)) - update_item_value(ctx, built_item, "dh:keyword2", fields.keyword2(json_item)) - update_item_value(ctx, built_item, "dh:keyword3", fields.keyword3(json_item)) - update_item_value(ctx, built_item, "dh:keyword4", fields.keyword4(json_item)) - update_item_value(ctx, built_item, "dh:keyword5", fields.keyword5(json_item)) - update_item_value(ctx, built_item, "dcterms:source", fields.source(json_item)) - update_item_value(ctx, built_item, "dcterms:medium", fields.medium(json_item)) - update_item_value(ctx, built_item, "dcterms:extent", fields.extent(json_item)) - update_item_value(ctx, built_item, "dcterms:language", fields.language(json_item)) - update_item_value(ctx, built_item, "dh:box", fields.box(json_item)) - update_item_value(ctx, built_item, "dh:folder", fields.folder(json_item)) - update_item_value(ctx, built_item, "foaf:name", fields.name(json_item)) - update_item_value(ctx, built_item, "dh:spatial_short_name", fields.spatial_short_name(json_item)) - update_item_value(ctx, built_item, "tei:correspSentName", fields.correspSentName(json_item)) - update_item_value(ctx, built_item, "tei:correspSentPlace", fields.correspSentPlace(json_item)) - update_item_value(ctx, built_item, "tei:correspSentDate", fields.correspSentDate(json_item), "numeric:timestamp") - update_item_value(ctx, built_item, "tei:correspDeliveredName", fields.correspDeliveredName(json_item)) - update_item_value(ctx, built_item, "tei:correspDeliveredPlace", fields.correspDeliveredPlace(json_item)) - update_item_value(ctx, built_item, "tei:correspDeliveredDate", fields.correspDeliveredDate(json_item), "numeric:timestamp") - update_item_value(ctx, built_item, "tei:distributor", fields.distributor(json_item)) - update_item_value(ctx, built_item, "tei:authority", fields.authority(json_item)) - update_item_value(ctx, built_item, "tei:biblNote", fields.biblNote(json_item)) - update_item_value(ctx, built_item, "dh:annotationsText", fields.annotationsText(json_item)) - update_item_value(ctx, built_item, "dh:itemText", fields.itemText(json_item)) - + for omeka_term, method_name, datatype in ctx.fields.field_manifest(): + value = getattr(ctx.fields, method_name)(row) + update_item_value(ctx, built_item, omeka_term, value, datatype) return built_item - except ValueError as e: - # A ValueError here means a field definition returned an unexpected - # type or structure. Log it and re-raise so the caller can record the - # error and skip this item. logger.error("ValueError building item dict: %s", e) raise -def link_item(ctx, json_item, existing_item): +def link_records(ctx, row, existing_item): """ Resolve inter-item relationships for a single item and attach them to the existing Omeka item dict. @@ -138,92 +68,67 @@ def link_item(ctx, json_item, existing_item): items — these are caught as KeyError or TypeError and logged at DEBUG level so they do not pollute the run log. Genuine API failures in link_item_record() will surface as exceptions and should be caught by the - caller (link_item in json_to_omeka.py). + caller in json_to_omeka.py. Parameters: * ctx - OmekaContext providing the API client and item_set_id - * json_item - raw JSON item dict from the Datura ES output + * row - raw JSON item dict from the Datura ES output * existing_item - the current Omeka item dict, deepcopied by the caller Returns the updated existing_item dict with relationship fields populated. + + To extend this function with additional linking logic (e.g. linking person + names to separate Omeka person-items), override link_records in + scripts/python/fieldoverrides.py, import and call this default first: + + from api_fields import link_records as default_link_records + + def link_records(ctx, row, existing_item): + existing_item = default_link_records(ctx, row, existing_item) + # ... collection-specific linking logic + return existing_item """ - # Each relationship field is optional; most items will not have all of - # them. Missing fields generate a DEBUG log entry, not a warning. + identifier = row.get("identifier") try: - part_ids = [part['id'] for part in json_item["has_part"]] + part_ids = [part['id'] for part in row["has_part"]] link_item_record(ctx, existing_item, "dcterms:hasPart", part_ids) except (KeyError, TypeError) as e: - logger.debug("No has_part data for %s: %s", json_item.get("identifier"), e) + logger.debug("No has_part data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dcterms:isPartOf", json_item["is_part_of"]["id"]) + link_item_record(ctx, existing_item, "dcterms:isPartOf", row["is_part_of"]["id"]) except (KeyError, TypeError) as e: - logger.debug("No is_part_of data for %s: %s", json_item.get("identifier"), e) + logger.debug("No is_part_of data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dcterms:relation", json_item["has_relation"]["id"]) + link_item_record(ctx, existing_item, "dcterms:relation", row["has_relation"]["id"]) except (KeyError, TypeError) as e: - logger.debug("No has_relation data for %s: %s", json_item.get("identifier"), e) + logger.debug("No has_relation data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dh:orderPrev", json_item["previous_item"]["id"]) + link_item_record(ctx, existing_item, "dh:orderPrev", row["previous_item"]["id"]) except (KeyError, TypeError) as e: - logger.debug("No previous_item data for %s: %s", json_item.get("identifier"), e) + logger.debug("No previous_item data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dh:orderNext", json_item["next_item"]["id"]) + link_item_record(ctx, existing_item, "dh:orderNext", row["next_item"]["id"]) except (KeyError, TypeError) as e: - logger.debug("No next_item data for %s: %s", json_item.get("identifier"), e) + logger.debug("No next_item data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "tei:correspNext", json_item["correspNext_omeka_s"]) + link_item_record(ctx, existing_item, "tei:correspNext", row["correspNext_omeka_s"]) except (KeyError, TypeError) as e: - logger.debug("No correspNext_omeka_s data for %s: %s", json_item.get("identifier"), e) + logger.debug("No correspNext_omeka_s data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "tei:correspPrev", json_item["correspPrev_omeka_s"]) + link_item_record(ctx, existing_item, "tei:correspPrev", row["correspPrev_omeka_s"]) except (KeyError, TypeError) as e: - logger.debug("No correspPrev_omeka_s data for %s: %s", json_item.get("identifier"), e) + logger.debug("No correspPrev_omeka_s data for %s: %s", identifier, e) return existing_item -def prepare_item(ctx, row, existing_item=None): - """ - Build a complete Omeka item dict from a Datura JSON record. - - Thin wrapper around build_item_dict() that provides the standard entry - point used by json_to_omeka.py for both new item creation and updates. - - Parameters: - * ctx - OmekaContext - * row - raw JSON item dict from the Datura ES output - * existing_item - existing Omeka item dict when updating, or None when - creating a new item - - Returns the built item dict, or raises ValueError if field extraction fails. - """ - return build_item_dict(ctx, row, existing_item) - - -def link_records(ctx, row, existing_item): - """ - Resolve and attach all relationship fields for a single item. - - Thin wrapper around link_item() that provides the standard entry point - used by json_to_omeka.py during the linking pass. - - Parameters: - * ctx - OmekaContext - * row - raw JSON item dict from the Datura ES output - * existing_item - the current Omeka item dict (deepcopied by the caller) - - Returns the updated item dict. - """ - return link_item(ctx, row, existing_item) - - def update_item_value(ctx, item, key, value, datatype="literal"): """ Set or replace a property on an Omeka item dict. @@ -407,4 +312,4 @@ def link_item_record(ctx, item, key, values, item_set=False, filter_property="dc item[key].append(formatted) - return item + return item \ No newline at end of file diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 2f4d31f8e..e38f72352 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -183,37 +183,27 @@ def delete_media_items(ctx, matching_item): OmekaMediaError("Unexpected error deleting media {}: {}".format(media_id, err)) ) - -def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): +def build_thumbnail_url(ctx, json_item): """ - Download a IIIF thumbnail and upload it to Omeka as the item's primary media. - - Thumbnail ingest is performed before HTML ingest so that Omeka designates - the image as the item's primary_media (Omeka S uses the first media object - as the primary). + Construct the remote IIIF URL and local cache filename for this item's thumbnail. - If the source JSON record has no cover_image field, the function returns - immediately — not all items have thumbnails. - - If the thumbnail cannot be downloaded (network error, 4xx/5xx from the - IIIF server) or if the upload to Omeka fails, the failure is logged and - the function returns — the HTML ingest still proceeds. + Returns a (remote_url, local_filename) tuple, or None if the item has no + cover_image. local_filename is a string suitable for joining with iiif_dir; + ingest_thumbnail() appends it to the iiif_dir path. Parameters: - * ctx - OmekaContext (provides iiif_server, client, item_set_id) - * json_item - dict: one record from a Datura ES JSON file - * matching_item - dict: the Omeka item to attach the thumbnail to - * iiif_dir - pathlib.Path pointing to the local IIIF output directory - where the downloaded thumbnail is cached temporarily + * ctx - OmekaContext (provides iiif_server, iiif_collection) + * json_item - dict: one record from a Datura ES JSON file + + To override URL construction for a collection (e.g. a different IIIF path + convention), define build_thumbnail_url in scripts/python/process_overrides.py. """ + collection_name = ctx.iiif_collection if ctx.iiif_collection else json_item.get("collection", "") cover_image = json_item.get("cover_image") - identifier = json_item.get("identifier", "unknown") if not cover_image: - # No thumbnail configured for this item — nothing to do. - logger.debug("No cover_image for %r; skipping thumbnail ingest", identifier) - return + return None # Parse any existing extension from the cover_image name. Image identifiers # often contain dots that are not extensions (e.g. loc.00001, ccda.let00001), @@ -228,7 +218,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): # Construct the IIIF Image API URL for the thumbnail. # The !200,200 size specifier requests a thumbnail that fits within a # 200×200 bounding box while preserving aspect ratio. - thumbnail_remote = ( + remote = ( "{}/iiif/2/{collection}%2F{image}{ext}/full/!200,200/0/default.jpg".format( ctx.iiif_server, collection=collection_name, @@ -238,7 +228,43 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): ) # Cache the thumbnail locally using the same URL-encoded filename so that # re-runs can be inspected on disk if needed. - thumbnail_local = iiif_dir / "{}%2F{}{}".format(collection_name, stem, image_ext) + local_name = "{}%2F{}{}".format(collection_name, stem, image_ext) + + return remote, local_name + + +def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): + """ + Download a IIIF thumbnail and upload it to Omeka as the item's primary media. + + Thumbnail ingest is performed before HTML ingest so that Omeka designates + the image as the item's primary_media (Omeka S uses the first media object + as the primary). + + URL construction is delegated to build_thumbnail_url(), which can be + overridden independently in scripts/python/process_overrides.py. + + If build_thumbnail_url() returns None (no cover_image on the item), or if + the download or upload fails, the function returns — the HTML ingest still + proceeds. + + Parameters: + * ctx - OmekaContext (provides iiif_server, client, item_set_id) + * json_item - dict: one record from a Datura ES JSON file + * matching_item - dict: the Omeka item to attach the thumbnail to + * iiif_dir - pathlib.Path pointing to the local IIIF output directory + where the downloaded thumbnail is cached temporarily + """ + identifier = json_item.get("identifier", "unknown") + + result = build_thumbnail_url(ctx, json_item) + if result is None: + # No thumbnail for this item — nothing to do. + logger.debug("No cover_image for %r; skipping thumbnail ingest", identifier) + return + + thumbnail_remote, local_name = result + thumbnail_local = iiif_dir / local_name # --- Download --- try: @@ -352,6 +378,30 @@ def ingest_html(ctx, json_item, matching_item, html_dir): # Main processing loop # --------------------------------------------------------------------------- +def ingest_item_media(ctx, json_item, matching_item, html_dir, iiif_dir): + """ + Run the full media pipeline for a single Omeka item. + + Encapsulates the delete-then-reingest sequence so that collections needing + a different media pipeline (e.g. adding a PDF step, skipping thumbnails for + certain item types) can override this function without touching the item-lookup and + skip logic in process_items(). + + Order matters: thumbnail must be uploaded before HTML so that Omeka + designates the image as primary_media (Omeka S uses the first media object + attached to an item as its primary). + + Parameters: + * ctx - OmekaContext + * json_item - dict: one record from a Datura ES JSON file + * matching_item - dict: the current Omeka item retrieved from the API + * html_dir - pathlib.Path to output//html/ + * iiif_dir - pathlib.Path to output//iiif/ + """ + delete_media_items(ctx, matching_item) + ingest_thumbnail(ctx, json_item, matching_item, iiif_dir) + ingest_html(ctx, json_item, matching_item, html_dir) + def process_items(ctx, pathlist, html_dir, iiif_dir): """ For each JSON record, look up the Omeka item and ingest its media. @@ -431,12 +481,8 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): continue # --- Media pipeline --- - # Delete first, then re-upload. Order matters: thumbnail must be - # uploaded before HTML so that Omeka designates the image as - # primary_media. - delete_media_items(ctx, matching_item) - ingest_thumbnail(ctx, json_item, matching_item, iiif_dir) - ingest_html(ctx, json_item, matching_item, html_dir) + ingest_item_media(ctx, json_item, matching_item, html_dir, iiif_dir) + # --------------------------------------------------------------------------- From 66fa30cd099a001dc2253a78069be3d50e72e61b Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 09:42:58 -0500 Subject: [PATCH 172/222] cleanup --- lib/datura/python/api_fields.py | 10 ---------- lib/datura/python/html_and_media_ingest.py | 2 -- 2 files changed, 12 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 900f64737..458295f78 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -77,16 +77,6 @@ def link_records(ctx, row, existing_item): Returns the updated existing_item dict with relationship fields populated. - To extend this function with additional linking logic (e.g. linking person - names to separate Omeka person-items), override link_records in - scripts/python/fieldoverrides.py, import and call this default first: - - from api_fields import link_records as default_link_records - - def link_records(ctx, row, existing_item): - existing_item = default_link_records(ctx, row, existing_item) - # ... collection-specific linking logic - return existing_item """ identifier = row.get("identifier") diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index e38f72352..07e9d86b7 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -195,8 +195,6 @@ def build_thumbnail_url(ctx, json_item): * ctx - OmekaContext (provides iiif_server, iiif_collection) * json_item - dict: one record from a Datura ES JSON file - To override URL construction for a collection (e.g. a different IIIF path - convention), define build_thumbnail_url in scripts/python/process_overrides.py. """ collection_name = ctx.iiif_collection if ctx.iiif_collection else json_item.get("collection", "") From 41649d5bfc2e9b562a4fcb9b89528a7f18ebe66a Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 14:07:28 -0500 Subject: [PATCH 173/222] cleanup --- lib/datura/python/api_fields.py | 3 +-- lib/datura/python/html_and_media_ingest.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 458295f78..c520ce54e 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -177,8 +177,7 @@ def add_formatted_value(ctx, item, key, value, datatype, label=""): if datatype == "literal": value = str(value) - # Look up the property ID via the cache — avoids one API round-trip per - # field per item across the entire run. + # Look up the property ID via the cache. prop_id = ctx.get_property_id(key) prop_value = { diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 07e9d86b7..b2ec0d5ee 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -240,8 +240,8 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): as the primary). URL construction is delegated to build_thumbnail_url(), which can be - overridden independently in scripts/python/process_overrides.py. - + overridden independently. + If build_thumbnail_url() returns None (no cover_image on the item), or if the download or upload fails, the function returns — the HTML ingest still proceeds. From c4267beda5e2caaec728f3d978c45aab34f63281 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 14:39:16 -0500 Subject: [PATCH 174/222] rename field overrides example file and create process overrides example file --- ..._example.py => field_overrides_example.py} | 2 +- .../python/process_overrides_example.py | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) rename lib/datura/python/{omeka_overrides_example.py => field_overrides_example.py} (96%) create mode 100644 lib/datura/python/process_overrides_example.py diff --git a/lib/datura/python/omeka_overrides_example.py b/lib/datura/python/field_overrides_example.py similarity index 96% rename from lib/datura/python/omeka_overrides_example.py rename to lib/datura/python/field_overrides_example.py index 7d98abf72..98bc03647 100644 --- a/lib/datura/python/omeka_overrides_example.py +++ b/lib/datura/python/field_overrides_example.py @@ -1,4 +1,4 @@ -#copy this file to omeka_overrides.py in your scripts/python directory. Edit the return values as needed +#copy this file to field_overrides.py in your scripts/python directory. Edit the return values as needed from field_definitions import FieldDefinitions diff --git a/lib/datura/python/process_overrides_example.py b/lib/datura/python/process_overrides_example.py new file mode 100644 index 000000000..ed05b0a52 --- /dev/null +++ b/lib/datura/python/process_overrides_example.py @@ -0,0 +1,56 @@ +# Copy this file to process_overrides.py in your scripts/python directory. +# Define only the functions whose behavior differs from the defaults in +# api_fields.py (link_records, update_item_value, link_item_record) and +# html_and_media_ingest.py (build_thumbnail_url). Functions not defined +# here fall back to the default implementations automatically. +# +# Each function must match the signature of its default counterpart exactly. +# To extend rather than replace a default, import it directly: +# +# from api_fields import link_item_record as default_link_item_record +# +# Overriding link_item_record here is automatically picked up by the default +# link_records without needing to override link_records as well. + + +# Pattern 1: override build_thumbnail_url to use a different IIIF path format +# def build_thumbnail_url(ctx, json_item): +# collection = ctx.iiif_collection or json_item.get("collection", "") +# cover_image = json_item.get("cover_image") +# if not cover_image: +# return None +# remote = "{}/iiif/3/{}/{}/full/200,/0/default.jpg".format( +# ctx.iiif_server, collection, cover_image +# ) +# local_name = "{}_{}".format(collection, cover_image) +# return remote, local_name + + +# Pattern 2: override link_item_record to use a custom filter property +# def link_item_record(ctx, item, key, values, item_set=False, filter_property="dcterms:identifier"): +# from api_fields import link_item_record as default_link_item_record +# # Use a collection-specific property for lookups instead of dcterms:identifier +# return default_link_item_record(ctx, item, key, values, item_set, filter_property="dh:slug") + + +# Pattern 3: override update_item_value to skip a field under certain conditions +# def update_item_value(ctx, item, key, value, datatype="literal"): +# from api_fields import update_item_value as default_update_item_value +# # Do not post empty string values for any field +# if value == "": +# value = None +# return default_update_item_value(ctx, item, key, value, datatype) + + +# Pattern 4: override link_records to add a custom relationship type +# def link_records(ctx, row, existing_item): +# from api_fields import link_records as default_link_records +# existing_item = default_link_records(ctx, row, existing_item) +# # Add collection-specific "dh:relatedProject" links +# try: +# from api_fields import link_item_record +# project_ids = [p["id"] for p in row["related_projects"]] +# link_item_record(ctx, existing_item, "dh:relatedProject", project_ids) +# except (KeyError, TypeError): +# pass +# return existing_item \ No newline at end of file From 1214975422189e141b1b8e082c783b614f562845 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 14:40:18 -0500 Subject: [PATCH 175/222] refactor to enable process overrides; use importlib instead of sys.path --- lib/datura/python/api_fields.py | 18 ++++---- lib/datura/python/field_definitions.py | 47 ++++++++++++------- lib/datura/python/html_and_media_ingest.py | 3 +- lib/datura/python/json_to_omeka.py | 3 +- lib/datura/python/omeka_context.py | 52 +++++++++++++++++++++- 5 files changed, 96 insertions(+), 27 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index c520ce54e..a76e93824 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -49,9 +49,10 @@ def prepare_item(ctx, row, existing_item=None): """ try: built_item = existing_item if existing_item else {} + _update = ctx._fn_update_item_value or update_item_value for omeka_term, method_name, datatype in ctx.fields.field_manifest(): value = getattr(ctx.fields, method_name)(row) - update_item_value(ctx, built_item, omeka_term, value, datatype) + _update(ctx, built_item, omeka_term, value, datatype) return built_item except ValueError as e: logger.error("ValueError building item dict: %s", e) @@ -79,40 +80,41 @@ def link_records(ctx, row, existing_item): """ identifier = row.get("identifier") + _link = ctx._fn_link_item_record or link_item_record try: part_ids = [part['id'] for part in row["has_part"]] - link_item_record(ctx, existing_item, "dcterms:hasPart", part_ids) + _link(ctx, existing_item, "dcterms:hasPart", part_ids) except (KeyError, TypeError) as e: logger.debug("No has_part data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dcterms:isPartOf", row["is_part_of"]["id"]) + _link(ctx, existing_item, "dcterms:isPartOf", row["is_part_of"]["id"]) except (KeyError, TypeError) as e: logger.debug("No is_part_of data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dcterms:relation", row["has_relation"]["id"]) + _link(ctx, existing_item, "dcterms:relation", row["has_relation"]["id"]) except (KeyError, TypeError) as e: logger.debug("No has_relation data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dh:orderPrev", row["previous_item"]["id"]) + _link(ctx, existing_item, "dh:orderPrev", row["previous_item"]["id"]) except (KeyError, TypeError) as e: logger.debug("No previous_item data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "dh:orderNext", row["next_item"]["id"]) + _link(ctx, existing_item, "dh:orderNext", row["next_item"]["id"]) except (KeyError, TypeError) as e: logger.debug("No next_item data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "tei:correspNext", row["correspNext_omeka_s"]) + _link(ctx, existing_item, "tei:correspNext", row["correspNext_omeka_s"]) except (KeyError, TypeError) as e: logger.debug("No correspNext_omeka_s data for %s: %s", identifier, e) try: - link_item_record(ctx, existing_item, "tei:correspPrev", row["correspPrev_omeka_s"]) + _link(ctx, existing_item, "tei:correspPrev", row["correspPrev_omeka_s"]) except (KeyError, TypeError) as e: logger.debug("No correspPrev_omeka_s data for %s: %s", identifier, e) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index c4fb902a7..92956f2f4 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -1,6 +1,7 @@ +import importlib.util import logging -import sys from datetime import datetime +from pathlib import Path logger = logging.getLogger(__name__) @@ -328,9 +329,10 @@ def get_fields(omeka_data_base=""): """ Return the appropriate FieldDefinitions instance for this collection. - Attempts to import CustomFields from scripts/python/omeka_overrides.py - in the collection directory. If that file does not exist, falls back to - the default FieldDefinitions class. + Looks for a CustomFields class in scripts/python/field_overrides.py in + the collection directory (resolved from the current working directory). + If that file does not exist, falls back to the default FieldDefinitions + class. If the file exists but cannot be loaded, raises RuntimeError. Parameters: * omeka_data_base - passed through to the FieldDefinitions constructor @@ -339,16 +341,29 @@ def get_fields(omeka_data_base=""): Returns a FieldDefinitions instance (or a CustomFields subclass of it). """ + override_path = Path.cwd() / "scripts" / "python" / "field_overrides.py" + if not override_path.exists(): + return FieldDefinitions(omeka_data_base=omeka_data_base) + try: - # Insert at position 0 so the collection's scripts/python directory - # takes precedence over any system-installed omeka_overrides module. - sys.path.insert(0, './scripts/python') - from omeka_overrides import CustomFields - # CustomFields inherits __init__ from FieldDefinitions, so - # omeka_data_base is passed through automatically. Override __init__ - # in CustomFields only if you need additional constructor logic. - logger.warning("Omeka overrides found at %s; custom field mappings will be applied.", "scripts/python/omeka_overrides.py") - return CustomFields(omeka_data_base=omeka_data_base) - except ImportError: - # No collection-specific overrides found; use the defaults. - return FieldDefinitions(omeka_data_base=omeka_data_base) \ No newline at end of file + spec = importlib.util.spec_from_file_location("field_overrides", override_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + except Exception as e: + raise RuntimeError( + "Failed to load field overrides from {}: {}".format(override_path, e) + ) from e + + CustomFields = getattr(module, "CustomFields", None) + if CustomFields is None: + # File exists but defines no CustomFields class — use defaults. + return FieldDefinitions(omeka_data_base=omeka_data_base) + + # CustomFields inherits __init__ from FieldDefinitions, so + # omeka_data_base is passed through automatically. Override __init__ + # in CustomFields only if you need additional constructor logic. + logger.warning( + "Field overrides found at %s; custom field mappings will be applied.", + override_path, + ) + return CustomFields(omeka_data_base=omeka_data_base) \ No newline at end of file diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index b2ec0d5ee..91d24c8dd 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -255,7 +255,8 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): """ identifier = json_item.get("identifier", "unknown") - result = build_thumbnail_url(ctx, json_item) + _build_thumbnail_url = ctx._fn_build_thumbnail_url or build_thumbnail_url + result = _build_thumbnail_url(ctx, json_item) if result is None: # No thumbnail for this item — nothing to do. logger.debug("No cover_image for %r; skipping thumbnail ingest", identifier) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 2bbcef0f6..d8517e0b4 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -414,7 +414,8 @@ def _link_item(ctx, json_item, matching_items): item_to_link = copy.deepcopy(matching_items["results"][0]) try: - linked_item = api_fields.link_records(ctx, json_item, item_to_link) + _link_records = ctx._fn_link_records or api_fields.link_records + linked_item = _link_records(ctx, json_item, item_to_link) except Exception as err: ctx.record_error(OmekaAPIError(item_id, "link_records", err)) return diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index ee34cd788..94b85a749 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -5,6 +5,7 @@ """ +import importlib.util import logging from logging.handlers import RotatingFileHandler import os @@ -407,10 +408,59 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, # ---- Field definitions -------------------------------------------- # Load collection-specific field mappings once here. get_fields() returns - # a CustomFields subclass if scripts/python/omeka_overrides.py is present; + # a CustomFields subclass if scripts/python/field_overrides.py is present; # otherwise the default FieldDefinitions instance. self.fields = get_fields(omeka_data_base=self.omeka_data_base) + # ---- Process function overrides ----------------------------------- + # Load collection-specific replacements for four pipeline functions: + # link_records, update_item_value, link_item_record (api_fields.py) + # build_thumbnail_url (html_and_media_ingest.py) + # + # Each attribute is None when no override is present; call sites + # resolve the default via: (ctx._fn_X or module.X)(ctx, ...) + # + # importlib.util is used to load the file by explicit path so that + # sys.path is not mutated and the module is not cached in sys.modules, + # keeping each context init independent. + self._fn_link_records = None + self._fn_update_item_value = None + self._fn_link_item_record = None + self._fn_build_thumbnail_url = None + + _process_override_path = Path.cwd() / "scripts" / "python" / "process_overrides.py" + if _process_override_path.exists(): + try: + _spec = importlib.util.spec_from_file_location( + "process_overrides", _process_override_path + ) + _po = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(_po) + self._fn_link_records = getattr(_po, "link_records", None) + self._fn_update_item_value = getattr(_po, "update_item_value", None) + self._fn_link_item_record = getattr(_po, "link_item_record", None) + self._fn_build_thumbnail_url = getattr(_po, "build_thumbnail_url", None) + _active = [ + n for n, f in [ + ("link_records", self._fn_link_records), + ("update_item_value", self._fn_update_item_value), + ("link_item_record", self._fn_link_item_record), + ("build_thumbnail_url", self._fn_build_thumbnail_url), + ] if f is not None + ] + if _active: + logger.warning( + "Process overrides found at %s; active overrides: %s", + _process_override_path, + _active, + ) + except Exception as e: + raise OmekaConfigError( + "Failed to load process overrides from {}: {}".format( + _process_override_path, e + ) + ) from e + # ----------------------------------------------------------------------- # Properties From 3df19a5f75e9621ea00568f69edcbc367ef755c4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 15:53:10 -0500 Subject: [PATCH 176/222] length-format file_handler log levelname to 8 chars --- lib/datura/python/omeka_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index ee34cd788..b629f2caf 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -59,7 +59,7 @@ def configure_logging(level="INFO"): file_handler = RotatingFileHandler( "logs/python.log", maxBytes=5 * 1024 * 1024, backupCount=3 ) - file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")) + file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)-8s] %(name)s: %(message)s")) file_handler.setLevel(logging.DEBUG) # always save everything to logs # Console handler — respects the requested level From eeb3aa8fe01d0cbcf0fd88ad75d86e909d45e708 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 16:05:12 -0500 Subject: [PATCH 177/222] add format_filter to doc comments --- lib/datura/python/omeka_context.py | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index b629f2caf..4061c8c8c 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -223,10 +223,12 @@ def from_args(cls, args): Parameters: * args - argparse.Namespace produced by an entrypoint's _parse_args(). Expected attributes: - .environment str "development" or "production" - .regex str optional file-filter pattern, or None - .update_time str optional date/time string for -u filter, or None - .media_skip bool skip re-ingesting existing media + .environment str "development" or "production" + .format_filter str optional format string for -f (directory-based) + filter, or None + .regex str optional file-filter pattern, or None + .update_time str optional date/time string for -u filter, or None + .media_skip bool skip re-ingesting existing media (html_and_media_ingest only; absent on json_to_omeka args, defaults to False) @@ -310,19 +312,21 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, this constructor directly except in tests. Parameters: - * env_config - merged dict: the "default" section of private.yml - overlaid with the environment-specific section so that - per-environment values take precedence over defaults. - Must contain omeka_server, key_identity, key_credential, - resource_template, omeka_data_base, and item_set. - * environment - "development" or "production" - * regex - optional regex string to filter input file paths; - None means process all files in the output directory - * media_skip - if True, items that already have 2+ media objects - (thumbnail + HTML) are skipped during media ingest - * update_time - optional datetime; if set, only items whose source file - mtime >= this value are processed (mirrors the -u flag - from the main Datura post command) + * env_config - merged dict: the "default" section of private.yml + overlaid with the environment-specific section so that + per-environment values take precedence over defaults. + Must contain omeka_server, key_identity, key_credential, + resource_template, omeka_data_base, and item_set. + * environment - "development" or "production" + * format_filter - optional format string to filter files by format (directory); + None means process all files in the output directory + * regex - optional regex string to filter input file paths; + None means process all files in the output directory + * media_skip - if True, items that already have 2+ media objects + (thumbnail + HTML) are skipped during media ingest + * update_time - optional datetime; if set, only items whose source file + mtime >= this value are processed (mirrors the -u flag + from the main Datura post command) """ # ---- Validate required config keys -------------------------------- # Validate up front so that failures are immediate and descriptive. From a01aae58b4d0b983765d085e069229941339404d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 16:20:52 -0500 Subject: [PATCH 178/222] add missing options to doc comments --- lib/datura/python/html_and_media_ingest.py | 10 ++++++---- lib/datura/python/json_to_omeka.py | 9 ++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 2f4d31f8e..4755428c4 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -71,10 +71,12 @@ def _parse_args(): Parse command-line arguments for the HTML/media ingest entrypoint. Returns an argparse.Namespace with: - * environment - "development" or "production" (default: "development") - * regex - optional file-filter pattern string, or None - * media_skip - bool; True skips items that already have 2+ media objects - * log_level - logging level string, default "INFO" + * environment - "development" or "production" (default: "development") + * format_filter - optional format string for -f (directory-based) filter, or None + * regex - optional file-filter pattern string, or None + * update_time - optional date/time string for -u filter, or None + * media_skip - bool; True skips items that already have 2+ media objects + * log_level - logging level string, default "INFO" """ parser = argparse.ArgumentParser( description="Attach HTML and IIIF thumbnail media to existing Omeka S items." diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 2bbcef0f6..7bb5738b1 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -72,9 +72,12 @@ def _parse_args(): Parse command-line arguments for the JSON-to-Omeka entrypoint. Returns an argparse.Namespace with: - * environment - "development" or "production" (default: "development") - * regex - optional file-filter pattern string, or None - * log_level - logging level string, default "INFO" + * environment - "development" or "production" (default: "development") + * format_filter - optional format string for -f (directory-based) filter, or None + * json_output - bool; True skips post to Omeka API and instead writes data to files as JSON + * regex - optional file-filter pattern string, or None + * update_time - optional date/time string for -u filter, or None + * log_level - logging level string, default "INFO" """ parser = argparse.ArgumentParser( From e37116e884792408c2b41961b5156364a18f0cc4 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 19 Jun 2026 17:00:40 -0500 Subject: [PATCH 179/222] update comments in reset_client to clarify rationale for client reset --- lib/datura/python/json_to_omeka.py | 3 ++- lib/datura/python/omeka_context.py | 9 ++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 7bb5738b1..0de552dba 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -449,7 +449,8 @@ def main(): 4. Discover JSON files under output//es/. 5. Apply regex filter if -r was passed. 6. Run pass 1 (post_items). - 7. Reset the API client between passes for a clean connection. + 7. Reset the API client between passes for a clean connection (see + ctx.reset_client() docstring for why this is required). 8. Run pass 2 (link_items). 9. Print run summary; exit 1 if any per-item errors were recorded, 0 if all items succeeded. diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 4061c8c8c..02e361b5f 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -467,11 +467,10 @@ def get_property_id(self, term): def reset_client(self): """ - Re-instantiate the authenticated API client with a fresh connection. - - Called in json_to_omeka.py between the item-posting pass and the - item-linking pass to obtain a clean session before the second round - of API requests. + Re-instantiate the authenticated API client with a fresh connection + to clear its HTTP response cache. (Otherwise Pass 2 would return the + stale responses from GET requests in Pass 1, per the OmekaAPIClient's + requests_cache session wrapper.) The property ID cache is intentionally preserved: term-to-ID mappings do not change between passes, so clearing and re-fetching them would From 6698764d94a92c4a4a023f4b49911e900d1ab5b7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 08:49:13 -0500 Subject: [PATCH 180/222] add csv_rows option to run_omeka_script helper --- lib/datura/helpers.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 1e86e9f2d..ce3997c09 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -159,6 +159,7 @@ def self.run_omeka_script(script_path, options) command = ["python3", script_path] command.append("-e", Shellwords.escape(options["environment"])) if options["environment"] command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] + command.append("-c", Shellwords.escape(options["csv_rows"])) if options["csv_rows"] command.append("-f", Shellwords.escape(options["format"])) if options["format"] command.append("-m") if options["media_skip"] command.append("-j") if options["json_output"] From 2d8ca93685ddd42be0b5066b5bf1e77d3d9d7eaa Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 09:45:04 -0500 Subject: [PATCH 181/222] shift .format() syntax to f-strings --- lib/datura/python/api_fields.py | 6 +-- lib/datura/python/field_definitions.py | 2 +- lib/datura/python/html_and_media_ingest.py | 39 +++++++---------- lib/datura/python/json_to_omeka.py | 18 ++++---- lib/datura/python/omeka_context.py | 49 +++++++++------------- 5 files changed, 45 insertions(+), 69 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index dd502b798..67c20bb66 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -401,9 +401,9 @@ def link_item_record(ctx, item, key, values, item_set=False, filter_property="dc if item_set: # The item-sets plugin requires these extra fields in addition # to what prepare_property_value generates. - formatted['@id'] = '{}/item_sets/{}'.format(ctx.client.api_url, omeka_id) - formatted['value_resource_id'] = omeka_id - formatted['value_resource_name'] = 'item_sets' + formatted["@id"] = f"{ctx.client.api_url}/item_sets/{omeka_id}" + formatted["value_resource_id"] = omeka_id + formatted["value_resource_name"] = "item_sets" item[key].append(formatted) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index fe4cdc073..6aead0788 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -54,7 +54,7 @@ def uriData(self, json): # Strip the original path and reconstruct the URI under the # collection's configured media base URL. filename = uri_data.split("/")[-1] - new_uri_data = "{}/{}".format(self._omeka_data_base, filename) + new_uri_data = f"{self._omeka_data_base}/{filename}" return new_uri_data def dcterms_type(self, json): diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 4755428c4..4bb52e66b 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -49,12 +49,10 @@ ) except ModuleNotFoundError as err: raise SystemExit( - "\033[31m" - "ERROR: {}\n" + f"\033[31m ERROR: {err}\n" "A required Python package could not be found. " "You may need to ensure the virtual environment is activated before running this script.\n" - "You may also need to be connected to the VPN." - "\033[0m".format(err) + "You may also need to be connected to the VPN.\033[0m" ) from err # Module-level logger. Records from this module appear as @@ -174,15 +172,11 @@ def delete_media_items(ctx, matching_item): ) else: ctx.record_error( - OmekaMediaError( - "HTTP {} deleting media {}: {}".format( - err.response.status_code, media_id, err - ) - ) + OmekaMediaError(f"HTTP {err.response.status_code} deleting media {media_id}: {err}") ) except Exception as err: ctx.record_error( - OmekaMediaError("Unexpected error deleting media {}: {}".format(media_id, err)) + OmekaMediaError(f"Unexpected error deleting media {media_id}: {err}") ) @@ -231,16 +225,11 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): # The !200,200 size specifier requests a thumbnail that fits within a # 200×200 bounding box while preserving aspect ratio. thumbnail_remote = ( - "{}/iiif/2/{collection}%2F{image}{ext}/full/!200,200/0/default.jpg".format( - ctx.iiif_server, - collection=collection_name, - image=stem, - ext=image_ext, - ) + f"{ctx.iiif_server}/iiif/2/{collection_name}%2F{stem}{image_ext}/full/!200,200/0/default.jpg" ) # Cache the thumbnail locally using the same URL-encoded filename so that # re-runs can be inspected on disk if needed. - thumbnail_local = iiif_dir / "{}%2F{}{}".format(collection_name, stem, image_ext) + thumbnail_local = iiif_dir / f"{collection_name}%2F{stem}{image_ext}" # --- Download --- try: @@ -285,7 +274,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): except Exception as err: ctx.record_error( OmekaMediaError( - "Error posting thumbnail for {!r}: {}".format(identifier, err) + f"Error posting thumbnail for {identifier!r}: {err}" ) ) @@ -310,7 +299,7 @@ def ingest_html(ctx, json_item, matching_item, html_dir): * html_dir - pathlib.Path pointing to the HTML output directory """ identifier = json_item.get("identifier", "unknown") - file_path = html_dir / "{}.html".format(identifier) + file_path = html_dir / f"{identifier}.html" try: with open(file_path, "r", encoding="utf-8") as file: @@ -345,7 +334,7 @@ def ingest_html(ctx, json_item, matching_item, html_dir): except Exception as err: ctx.record_error( OmekaMediaError( - "Error posting HTML for {!r}: {}".format(identifier, err) + f"Error posting HTML for {identifier!r}: {err}" ) ) @@ -469,9 +458,9 @@ def main(): # Resolve all three environment-specific directories using the requested # environment so that -e production reads from output/production/ rather # than always defaulting to output/development/. - json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) - html_dir = ctx.resolve_path("output/{}/html".format(ctx.environment)) - iiif_dir = ctx.resolve_path("output/{}/iiif".format(ctx.environment)) + json_dir = ctx.resolve_path(f"output/{ctx.environment}/es") + html_dir = ctx.resolve_path(f"output/{ctx.environment}/html") + iiif_dir = ctx.resolve_path(f"output/{ctx.environment}/iiif") pathlist = list(Path(json_dir).glob("**/*.json")) @@ -485,7 +474,7 @@ def main(): logger.info( "Found %d JSON file(s) in %s (environment=%r, media_skip=%s)", len(pathlist), - "output/{}/es".format(ctx.environment), + f"output/{ctx.environment}/es", ctx.environment, ctx.media_skip, ) @@ -503,5 +492,5 @@ def main(): sys.exit(1) except OmekaConfigError as err: logger.debug("Fatal configuration error:", exc_info=True) - print("ERROR: {}".format(err), file=sys.stderr) + print(f"ERROR: {err}", file=sys.stderr) sys.exit(1) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 0de552dba..92b3bf07f 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -50,12 +50,10 @@ from omeka import filter_items, filter_items_by_date, filter_items_by_format, prepare_item_payload_using_template except ModuleNotFoundError as err: raise SystemExit( - "\033[31m" - "ERROR: {}\n" + f"\033[31m ERROR: {err}\n" "A required Python package could not be found. " "You may need to ensure the virtual environment is activated before running this script.\n" - "You may also need to be connected to the VPN." - "\033[0m".format(err) + "You may also need to be connected to the VPN.\033[0m" ) from err # Module-level logger. Records from this module appear as "json_to_omeka" @@ -192,8 +190,8 @@ def post_items(ctx, pathlist, json_output_dir=None): logger.warning("Could not prepare payload for %r; skipping", identifier) continue payload = prepare_item_payload_using_template(ctx, new_item, template_number) - out_path = json_output_dir / "{}.json".format(identifier) - relative_path = "output/{}/{}.json".format(ctx.environment, identifier) + out_path = json_output_dir / f"{identifier}.json" + relative_path = f"output/{ctx.environment}/{identifier}.json" logger.info("Writing Omeka payload for %r to %s", identifier, relative_path) with open(out_path, "w") as f: json.dump(payload, f, indent=2) @@ -472,7 +470,7 @@ def main(): # ctx.resolve_path() returns an absolute Path relative to cwd (the # collection root), so passing -e production reads from output/production/ # rather than always defaulting to output/development/. - json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) + json_dir = ctx.resolve_path(f"output/{ctx.environment}/es") pathlist = list(Path(json_dir).glob("**/*.json")) if ctx.format_filter: @@ -485,13 +483,13 @@ def main(): logger.info( "Found %d JSON file(s) in %s (environment=%r)", len(pathlist), - "output/{}/es".format(ctx.environment), + f"output/{ctx.environment}/es", ctx.environment, ) # --- JSON output mode (-j / --json-output) --- if args.json_output: - relative_dir = "output/{}/omeka".format(ctx.environment) + relative_dir = f"output/{ctx.environment}/omeka" omeka_out_dir = ctx.resolve_path(relative_dir) Path(omeka_out_dir).mkdir(parents=True, exist_ok=True) logger.info( @@ -528,5 +526,5 @@ def main(): sys.exit(1) except OmekaConfigError as err: logger.debug("Fatal configuration error:", exc_info=True) - print("ERROR: {}".format(err), file=sys.stderr) + print(f"ERROR: {err}", file=sys.stderr) sys.exit(1) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 02e361b5f..8daf1bdbc 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -145,7 +145,7 @@ def __init__(self, identifier, operation, cause): self.operation = operation self.cause = cause super().__init__( - "{} failed for {!r}: {}".format(operation, identifier, cause) + f"{operation} failed for {identifier!r}: {cause}" ) @@ -183,11 +183,9 @@ def parse_update_time(s): return datetime.strptime(s, fmt) except ValueError: continue - raise OmekaConfigError(RED + - "Invalid --update value {!r}. " - "Expected 'today', a date (2015-01-01), or date-time (2015-01-01T18:24)." - .format(s) - + RESET + raise OmekaConfigError( + f"{RED}Invalid --update value {s!r}. " + f"Expected 'today', a date (2015-01-01), or date-time (2015-01-01T18:24).{RESET}" ) @@ -283,25 +281,20 @@ def _load_config(path, env): with open(path) as f: contents = yaml.safe_load(f) except FileNotFoundError: - raise OmekaConfigError(RED + - "Config file not found: {}. " + raise OmekaConfigError( + f"{RED}Config file not found: {path}. " "Ensure config/private.yml exists in the collection directory " - "and that you are running the script from the collection root." - .format(path) - + RESET + f"and that you are running the script from the collection root.{RESET}" ) except yaml.YAMLError as exc: - raise OmekaConfigError(RED + - "Could not parse YAML in {}: {}".format(path, exc) - + RESET + raise OmekaConfigError( + f"{RED}Could not parse YAML in {path}: {exc}{RESET}" ) if env not in contents: raise OmekaConfigError(RED + - "Environment section {!r} not found in {}. " - "Available sections: {}" - .format(env, path, list(contents.keys())) - + RESET + f"{RED}Environment section {env!r} not found in {path}. " + f"Available sections: {list(contents.keys())}{RESET}" ) return contents[env] @@ -340,24 +333,20 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, ] missing_keys = [key for key in required_keys if key not in env_config] if missing_keys: - raise OmekaConfigError(RED + - "Missing required config key(s): {}. " - "Check the 'default' or {!r} section of config/private.yml." - .format(missing_keys, environment) - + RESET + raise OmekaConfigError( + f"{RED}Missing required config key(s): {missing_keys}. " + f"Check the 'default' or {environment!r} section of config/private.yml.{RESET}" ) # ---- Validate environment-specific item_set --------------------------- if "item_set" not in env_config: - raise OmekaConfigError(RED + - "Missing 'item_set' for environment {!r} in config/private.yml.\n" + raise OmekaConfigError( + f"{RED}Missing 'item_set' for environment {environment!r} in config/private.yml.\n" "Add the item set ID for this environment before running. Example:\n\n" - " {}:\n" + f" {environment}:\n" " item_set: 123\n\n" "To find your item set ID, log into the Omeka S admin and navigate " - "to Items > Item Sets." - .format(environment, environment) - + RESET + f"to Items > Item Sets.{RESET}" ) # ---- Runtime flags ------------------------------------------------ @@ -493,7 +482,7 @@ def resolve_path(self, relative): Resolve a path relative to the current working directory (collection root). Callers interpolate the environment into the path template: - json_dir = ctx.resolve_path("output/{}/es".format(ctx.environment)) + json_dir = ctx.resolve_path(f"output/{ctx.environment}/es") This ensures that passing -e production reads from output/production/ rather than always using output/development/. From 6620b66cc9b38709aeef66761c05ec9aae1e42f2 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 10:08:51 -0500 Subject: [PATCH 182/222] shift .format() syntax to f-strings in overrides file --- lib/datura/python/process_overrides_example.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/datura/python/process_overrides_example.py b/lib/datura/python/process_overrides_example.py index ed05b0a52..e9a0fa2c2 100644 --- a/lib/datura/python/process_overrides_example.py +++ b/lib/datura/python/process_overrides_example.py @@ -19,10 +19,8 @@ # cover_image = json_item.get("cover_image") # if not cover_image: # return None -# remote = "{}/iiif/3/{}/{}/full/200,/0/default.jpg".format( -# ctx.iiif_server, collection, cover_image -# ) -# local_name = "{}_{}".format(collection, cover_image) +# remote = f"{ctx.iiif_server}/iiif/3/{collection}/{cover_image}/full/200,/0/default.jpg" +# local_name = f"{collection}_{cover_image}" # return remote, local_name From aae06182327791d7ae586f57281a86c9c843ff0c Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 10:14:04 -0500 Subject: [PATCH 183/222] shift .format() syntax to f-strings in overrides-related error msgs --- lib/datura/python/field_definitions.py | 2 +- lib/datura/python/omeka_context.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 92956f2f4..f724eb645 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -351,7 +351,7 @@ def get_fields(omeka_data_base=""): spec.loader.exec_module(module) except Exception as e: raise RuntimeError( - "Failed to load field overrides from {}: {}".format(override_path, e) + f"Failed to load field overrides from {override_path}: {e}" ) from e CustomFields = getattr(module, "CustomFields", None) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 94b85a749..01c70e9c9 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -456,9 +456,7 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, ) except Exception as e: raise OmekaConfigError( - "Failed to load process overrides from {}: {}".format( - _process_override_path, e - ) + f"Failed to load process overrides from {_process_override_path}: {e}" ) from e From 93656d6ee48559d0c2f12a4adace0c1648bf09c8 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 13:40:21 -0500 Subject: [PATCH 184/222] fix iiif path in overrides examples --- lib/datura/python/process_overrides_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/datura/python/process_overrides_example.py b/lib/datura/python/process_overrides_example.py index e9a0fa2c2..90bcf4ccf 100644 --- a/lib/datura/python/process_overrides_example.py +++ b/lib/datura/python/process_overrides_example.py @@ -19,7 +19,7 @@ # cover_image = json_item.get("cover_image") # if not cover_image: # return None -# remote = f"{ctx.iiif_server}/iiif/3/{collection}/{cover_image}/full/200,/0/default.jpg" +# remote = f"{ctx.iiif_server}/iiif/2/{collection}%2F{cover_image}/full/!150,150/0/default.jpg" # local_name = f"{collection}_{cover_image}" # return remote, local_name From ca4076342874e196b29955dfd9f2da9a680f7545 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 14:22:11 -0500 Subject: [PATCH 185/222] account for dicts in person, contributor, and creator fields --- lib/datura/python/api_fields.py | 20 +++++++++++++++++--- lib/datura/python/field_definitions.py | 18 ++++++++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 67c20bb66..4d8f9af51 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -252,10 +252,24 @@ def update_item_value(ctx, item, key, value, datatype="literal"): if isinstance(value, (str, int, float)): item = add_formatted_value(ctx, item, key, value, datatype) elif isinstance(value, list): - # Deduplicate (preserving insertion order) and remove None entries. - value = list(dict.fromkeys(v for v in value if v is not None)) + # List entries may be strings or dicts (e.g. contributor returns + # [{"name": "...", "id": "..."}]). For dicts, the dedup key compares + # based on id when id is present or creates a sorted tuple of all + # items so that two dicts are only considered duplicates when every + # key-value pair matches. + seen = {} for v in value: - item = add_formatted_value(ctx, item, key, v, datatype) + if v is None: + continue + dedup_key = v.get("id") or tuple(sorted(v.items())) if isinstance(v,dict) else v + if dedup_key not in seen: + seen[dedup_key] = v + for v in seen.values(): + display = v.get("name") if isinstance(v,dict) else v + if display is None: + continue + else: + item = add_formatted_value(ctx, item, key, display, datatype) def add_formatted_value(ctx, item, key, value, datatype, label=""): diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 6aead0788..025e567c2 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -62,12 +62,16 @@ def dcterms_type(self, json): return json.get("type", None) def creator(self, json): - creator_names = [creator['name'] for creator in json.get("creator") or [] if 'name' in creator] - return creator_names + # Return dicts with name+id so the dedup in api_fields preserves + # entries that share a name but have different ids. + return [{"name": creator["name"], "id": creator.get("id", "")} + for creator in json.get("creator") or [] if "name" in creator] def contributor(self, json): - contributor_names = [contributor['name'] for contributor in json.get("contributor") or [] if 'name' in contributor] - return contributor_names + # Return dicts with name+id so the dedup in api_fields preserves + # entries that share a name but have different ids. + return [{"name": contrib["name"], "id": contrib.get("id", "")} + for contrib in json.get("contributor") or [] if "name" in contrib] # NOTE: use Pattern 5 in omeka_overrides if automatic conversion of dates with year or month only to yyyy-01-01 # back to yyyy for Omeka is desired @@ -196,8 +200,10 @@ def folder(self, json): return json.get("container_folder", None) def name(self, json): - person_names = [person['name'] for person in json.get("person") or [] if 'name' in person] - return person_names + # Return dicts with name+id so the dedup in api_fields preserves + # entries that share a name but have different ids. + return [{"name": person["name"], "id": person.get("id", "")} + for person in json.get("person") or [] if "name" in person] def spatial_short_name(self, json): spatial = json.get("spatial") From 4f358b82f477a067ec2016c4580859c514c90cf2 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 14:26:18 -0500 Subject: [PATCH 186/222] remove omeka_s fields from core datura; these should be worked into API schema fields or created as overrides --- lib/datura/python/field_definitions.py | 27 -------------------------- 1 file changed, 27 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 025e567c2..9b3783aa5 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -212,33 +212,6 @@ def spatial_short_name(self, json): places = [spatial] if isinstance(spatial, dict) else spatial short_names = [place['short_name'] for place in places if 'short_name' in place] return short_names - - def correspSentName(self, json): - return json.get("correspSentName_omeka_s", None) - - def correspSentPlace(self, json): - return json.get("correspSentPlace_omeka_s", None) - - def correspSentDate(self, json): - return json.get("correspSentDate_omeka_s", None) - - def correspDeliveredName(self, json): - return json.get("correspDeliveredName_omeka_s", None) - - def correspDeliveredPlace(self, json): - return json.get("correspDeliveredPlace_omeka_s", None) - - def correspDeliveredDate(self, json): - return json.get("correspDeliveredDate_omeka_s", None) - - def distributor(self, json): - return json.get("distributor_omeka_s", None) - - def authority(self, json): - return json.get("authority_omeka_s", None) - - def biblNote(self, json): - return json.get("biblNote_omeka_s", None) def annotationsText(self, json): return json.get("annotations_text", None) From d8323c363ca1815c6ae5367a628286451c9c702a Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 14:31:33 -0500 Subject: [PATCH 187/222] allow get_omeka_ids to search across item sets if none is passed --- lib/datura/python/api_fields.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 4d8f9af51..58b6cff6f 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -313,13 +313,13 @@ def add_formatted_value(ctx, item, key, value, datatype, label=""): return item -def get_omeka_ids(ctx, lookup_values, filter_property): +def get_omeka_ids(ctx, lookup_values, filter_property, item_set_id="ctx_default"): """ Resolve a list of lookup values to Omeka numeric item IDs. - For each lookup value, queries the Omeka API to find the matching item - within the configured item set. Used during the linking pass to convert - CDRH identifiers into the Omeka IDs required for resource:item links. + For each lookup value, queries the Omeka API to find the matching item. + Used during the linking pass to convert CDRH identifiers into the Omeka IDs + required for resource:item links. Parameters: * ctx - OmekaContext providing the API client and item_set_id @@ -328,12 +328,20 @@ def get_omeka_ids(ctx, lookup_values, filter_property): when filter_property is "o:id" * filter_property - the Omeka property to match against, e.g. "dcterms:identifier" or "o:id" + * item_set_id - restricts the search to a specific Omeka item set. + Defaults to ctx.item_set_id (the current collection). + Pass None to search across all item sets — useful when + the target items (e.g. a personography) live in a + separate item set from the collection being ingested. Returns a list of integer Omeka item IDs for all successfully resolved values. Logs a warning for values that cannot be resolved. """ omeka_ids = [] + # Resolve the sentinel to ctx.item_set_id so existing callers are unaffected. + resolved_item_set_id = ctx.item_set_id if item_set_id == "ctx_default" else item_set_id + # Normalise a single value to a list for uniform iteration. lookup_values = [lookup_values] if not isinstance(lookup_values, list) else lookup_values @@ -350,7 +358,7 @@ def get_omeka_ids(ctx, lookup_values, filter_property): match = ctx.client.filter_items_by_property( filter_property=filter_property, filter_value=lookup_value, - item_set_id=ctx.item_set_id, + item_set_id=resolved_item_set_id, ) if match["total_results"] >= 1: if match["total_results"] > 1: From 295c651009a2b3b5271d62e5eab78e7e5b457023 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 22 Jun 2026 15:25:10 -0500 Subject: [PATCH 188/222] remove omeka_s fields from manifest --- lib/datura/python/field_definitions.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index f724eb645..0239a12b0 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -96,15 +96,6 @@ def field_manifest(self): ("dh:folder", "folder", "literal"), ("foaf:name", "name", "literal"), ("dh:spatial_short_name", "spatial_short_name", "literal"), - ("tei:correspSentName", "correspSentName", "literal"), - ("tei:correspSentPlace", "correspSentPlace", "literal"), - ("tei:correspSentDate", "correspSentDate", "numeric:timestamp"), - ("tei:correspDeliveredName", "correspDeliveredName", "literal"), - ("tei:correspDeliveredPlace", "correspDeliveredPlace","literal"), - ("tei:correspDeliveredDate", "correspDeliveredDate", "numeric:timestamp"), - ("tei:distributor", "distributor", "literal"), - ("tei:authority", "authority", "literal"), - ("tei:biblNote", "biblNote", "literal"), ("dh:annotationsText", "annotationsText", "literal"), ("dh:itemText", "itemText", "literal"), ] From f3f09e45f658e8b5c669ae81605bfd58aa829177 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 13:52:55 -0500 Subject: [PATCH 189/222] rename row as json_item --- lib/datura/python/api_fields.py | 30 +++++++++++++------------- lib/datura/python/field_definitions.py | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index 21183d590..200360eea 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -6,8 +6,8 @@ CDRH identifiers in the live Omeka instance. The two primary entry points called by json_to_omeka.py are: - prepare_item(ctx, row, existing_item) — build or update item metadata - link_records(ctx, row, existing_item) — resolve and attach relationships + prepare_item(ctx, json_item, existing_item) — build or update item metadata + link_records(ctx, json_item, existing_item) — resolve and attach relationships All functions that need API access or configuration now receive a ctx (OmekaContext) parameter. The property ID cache on ctx (ctx.get_property_id()) @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) -def prepare_item(ctx, row, existing_item=None): +def prepare_item(ctx, json_item, existing_item=None): """ Build a complete Omeka item dict from a Datura JSON record. @@ -39,7 +39,7 @@ def prepare_item(ctx, row, existing_item=None): Parameters: * ctx - OmekaContext providing config and the property ID cache - * row - raw JSON item dict from the Datura ES output + * json_item - raw JSON item dict from the Datura ES output * existing_item - existing Omeka item dict to update in-place, or None when creating a new item (an empty dict is used instead) @@ -51,7 +51,7 @@ def prepare_item(ctx, row, existing_item=None): built_item = existing_item if existing_item else {} _update = ctx._fn_update_item_value or update_item_value for omeka_term, method_name, datatype in ctx.fields.field_manifest(): - value = getattr(ctx.fields, method_name)(row) + value = getattr(ctx.fields, method_name)(json_item) _update(ctx, built_item, omeka_term, value, datatype) return built_item except ValueError as e: @@ -59,7 +59,7 @@ def prepare_item(ctx, row, existing_item=None): raise -def link_records(ctx, row, existing_item): +def link_records(ctx, json_item, existing_item): """ Resolve inter-item relationships for a single item and attach them to the existing Omeka item dict. @@ -73,48 +73,48 @@ def link_records(ctx, row, existing_item): Parameters: * ctx - OmekaContext providing the API client and item_set_id - * row - raw JSON item dict from the Datura ES output + * json_item - raw JSON item dict from the Datura ES output * existing_item - the current Omeka item dict, deepcopied by the caller Returns the updated existing_item dict with relationship fields populated. """ - identifier = row.get("identifier") + identifier = json_item.get("identifier") _link = ctx._fn_link_item_record or link_item_record try: - part_ids = [part['id'] for part in row["has_part"]] + part_ids = [part['id'] for part in json_item["has_part"]] _link(ctx, existing_item, "dcterms:hasPart", part_ids) except (KeyError, TypeError) as e: logger.debug("No has_part data for %s: %s", identifier, e) try: - _link(ctx, existing_item, "dcterms:isPartOf", row["is_part_of"]["id"]) + _link(ctx, existing_item, "dcterms:isPartOf", json_item["is_part_of"]["id"]) except (KeyError, TypeError) as e: logger.debug("No is_part_of data for %s: %s", identifier, e) try: - _link(ctx, existing_item, "dcterms:relation", row["has_relation"]["id"]) + _link(ctx, existing_item, "dcterms:relation", json_item["has_relation"]["id"]) except (KeyError, TypeError) as e: logger.debug("No has_relation data for %s: %s", identifier, e) try: - _link(ctx, existing_item, "dh:orderPrev", row["previous_item"]["id"]) + _link(ctx, existing_item, "dh:orderPrev", json_item["previous_item"]["id"]) except (KeyError, TypeError) as e: logger.debug("No previous_item data for %s: %s", identifier, e) try: - _link(ctx, existing_item, "dh:orderNext", row["next_item"]["id"]) + _link(ctx, existing_item, "dh:orderNext", json_item["next_item"]["id"]) except (KeyError, TypeError) as e: logger.debug("No next_item data for %s: %s", identifier, e) try: - _link(ctx, existing_item, "tei:correspNext", row["correspNext_omeka_s"]) + _link(ctx, existing_item, "tei:correspNext", json_item["correspNext_omeka_s"]) except (KeyError, TypeError) as e: logger.debug("No correspNext_omeka_s data for %s: %s", identifier, e) try: - _link(ctx, existing_item, "tei:correspPrev", row["correspPrev_omeka_s"]) + _link(ctx, existing_item, "tei:correspPrev", json_item["correspPrev_omeka_s"]) except (KeyError, TypeError) as e: logger.debug("No correspPrev_omeka_s data for %s: %s", identifier, e) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 179680c34..1809babab 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -41,7 +41,7 @@ def field_manifest(self): * datatype - Omeka data type string prepare_item() in api_fields.py iterates this manifest to build the - item payload, calling getattr(ctx.fields, method_name)(row) for each + item payload, calling getattr(ctx.fields, method_name)(json_item) for each entry. Overriding field_manifest in a CustomFields subclass is the recommended way to add collection-specific Omeka properties without touching prepare_item() itself. From ecbcdb6e73d0fcf4e22d00f6d528855e951e48c0 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 14:13:12 -0500 Subject: [PATCH 190/222] organize options alphabetically --- bin/post_omeka | 12 ++++++------ bin/post_omeka_html | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index 5096b57d2..a35d20f78 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -11,8 +11,10 @@ options = {} optparse = OptionParser.new do |opts| # Set a banner opts.banner = @usage - opts.on('-s', '--skip', 'skip generation step and just post to Omeka') do - generate_es = false + opts.on('-c', '--csv-rows [input]', 'only process CSV rows matching this identifier regex') do |input| + if input && input.length > 0 + options["csv_rows"] = input + end end opts.on('-e', '--environment [input]', 'environment (default: development, or production)') do |input| @@ -37,10 +39,8 @@ optparse = OptionParser.new do |opts| end end - opts.on('-c', '--csv-rows [input]', 'only process CSV rows matching this identifier regex') do |input| - if input && input.length > 0 - options["csv_rows"] = input - end + opts.on('-s', '--skip', 'skip generation step and just post to Omeka') do + generate_es = false end opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| diff --git a/bin/post_omeka_html b/bin/post_omeka_html index c23fe3e48..6074a4e00 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -7,12 +7,10 @@ options = {} optparse = OptionParser.new do |opts| # Set a banner opts.banner = @usage - opts.on('-s', '--skip', 'skip generation step and just post to Omeka') do - generate_es = false - end - - opts.on('-m', '--media_skip', 'skip deleting and regenerating media') do - options["media_skip"] = true + opts.on('-c', '--csv-rows [input]', 'only process CSV rows matching this identifier regex') do |input| + if input && input.length > 0 + options["csv_rows"] = input + end end opts.on('-e', '--environment [input]', 'environment (default: development, or production)') do |input| @@ -27,16 +25,18 @@ optparse = OptionParser.new do |opts| end end + opts.on('-m', '--media_skip', 'skip deleting and regenerating media') do + options["media_skip"] = true + end + opts.on('-r', '--regex [input]', 'only generate and post files matching this regex') do |input| if input && input.length > 0 options["regex"] = input end end - opts.on('-c', '--csv-rows [input]', 'only process CSV rows matching this identifier regex') do |input| - if input && input.length > 0 - options["csv_rows"] = input - end + opts.on('-s', '--skip', 'skip generation step and just post to Omeka') do + generate_es = false end opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| From faf98ef2e88d13b22bc2bddccbcb3ea17a2f956d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 14:34:15 -0500 Subject: [PATCH 191/222] add p and u to bin files --- bin/post_omeka | 45 +++++++++++++++++++++++++++++++++++++++++++++ bin/post_omeka_html | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/bin/post_omeka b/bin/post_omeka index a35d20f78..ed226b109 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -33,6 +33,10 @@ optparse = OptionParser.new do |opts| options["json_output"] = true end + opts.on('-p', '--proceed [input]', 'proceed from the file matching this regex, or from the last checkpoint if no value given') do |input| + options["proceed"] = input + end + opts.on('-r', '--regex [input]', 'only generate and post files matching this regex') do |input| if input && input.length > 0 options["regex"] = input @@ -43,6 +47,12 @@ optparse = OptionParser.new do |opts| generate_es = false end + opts.on('-u', '--update [input]', 'only post files updated after this date (today, 2015-01-01, or 2015-01-01T18:24)') do |input| + if input && input.length > 0 + options["update_time"] = input + end + end + opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| #does nothing, this is a placeholder to catch all arguments not needed now and pass them along to the main script end @@ -52,6 +62,41 @@ optparse.parse(ARGV) #remove options not used in the main script ARGV.delete("-s") ARGV.delete("--skip") +#-p checkpointing belongs to the Omeka posting step (Python); strip it so +#DataManager does not trigger the Ruby proceed/checkpoint logic instead +ARGV.delete("-p") +ARGV.delete("--proceed") +ARGV.reject! { |arg| arg.start_with?("--proceed=") || (arg.start_with?("-p") && arg.length > 2) } +#if -p was given, resolve the proceed value from the Python checkpoint and +#re-inject into ARGV so DataManager also starts from the same file, preventing +#all source files from being transformed unnecessarily +if options.key?("proceed") + proceed_value = options["proceed"] + if proceed_value.nil? + env = options["environment"] || "development" + checkpoint_file = File.join("logs", "proceed_omeka_#{env}") + if File.exist?(checkpoint_file) + last = File.read(checkpoint_file).strip + unless last.empty? + print "Continue from #{last}? (y/n): " + $stdout.flush + response = $stdin.gets&.chomp&.downcase || "" + if response == "y" + proceed_value = last + else + puts "Exiting." + exit 0 + end + end + end + if proceed_value.nil? + puts "ERROR: --proceed given with no value but no checkpoint file found at #{checkpoint_file}" + exit 1 + end + end + ARGV.push("--proceed=#{proceed_value}") + options["proceed"] = proceed_value +end ARGV.delete("-j") ARGV.delete("--json-output") #add options to output a json file instead of posting it to Elasticsearch diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 6074a4e00..2ea697c61 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -29,6 +29,10 @@ optparse = OptionParser.new do |opts| options["media_skip"] = true end + opts.on('-p', '--proceed [input]', 'proceed from the file matching this regex, or from the last checkpoint if no value given') do |input| + options["proceed"] = input + end + opts.on('-r', '--regex [input]', 'only generate and post files matching this regex') do |input| if input && input.length > 0 options["regex"] = input @@ -39,6 +43,12 @@ optparse = OptionParser.new do |opts| generate_es = false end + opts.on('-u', '--update [input]', 'only post files updated after this date (today, 2015-01-01, or 2015-01-01T18:24)') do |input| + if input && input.length > 0 + options["update_time"] = input + end + end + opts.on('-[!-~]', '-[!-~] [input]', 'arguments for the main Datura script') do |name, value| #does nothing, this is a placeholder to catch all arguments not needed now and pass them along to the main script end @@ -50,6 +60,41 @@ ARGV.delete("-m") ARGV.delete("--media_skip") ARGV.delete("-s") ARGV.delete("--skip") +#-p checkpointing belongs to the Omeka posting step (Python); strip it so +#DataManager does not trigger the Ruby proceed/checkpoint logic instead +ARGV.delete("-p") +ARGV.delete("--proceed") +ARGV.reject! { |arg| arg.start_with?("--proceed=") || (arg.start_with?("-p") && arg.length > 2) } +#if -p was given, resolve the proceed value from the Python checkpoint and +#re-inject into ARGV so DataManager also starts from the same file, preventing +#all source files (including CSVs) from being transformed unnecessarily +if options.key?("proceed") + proceed_value = options["proceed"] + if proceed_value.nil? + env = options["environment"] || "development" + checkpoint_file = File.join("logs", "proceed_omeka_html_#{env}") + if File.exist?(checkpoint_file) + last = File.read(checkpoint_file).strip + unless last.empty? + print "Continue from #{last}? (y/n): " + $stdout.flush + response = $stdin.gets&.chomp&.downcase || "" + if response == "y" + proceed_value = last + else + puts "Exiting." + exit 0 + end + end + end + if proceed_value.nil? + puts "ERROR: --proceed given with no value but no checkpoint file found at #{checkpoint_file}" + exit 1 + end + end + ARGV.push("--proceed=#{proceed_value}") + options["proceed"] = proceed_value +end #add option to generate html ARGV.unshift("-x", "html") #create DataManager before conditional run From 49771d3ccb15e5888abad55bebd928baa124f58d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 14:37:24 -0500 Subject: [PATCH 192/222] extend and reorder run_omeka_script helper --- lib/datura/helpers.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/datura/helpers.rb b/lib/datura/helpers.rb index 789fd65b4..baf9f9724 100644 --- a/lib/datura/helpers.rb +++ b/lib/datura/helpers.rb @@ -258,11 +258,18 @@ def self.run_omeka_script(script_path, options) return end command = ["python3", script_path] - command.append("-e", Shellwords.escape(options["environment"])) if options["environment"] - command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] command.append("-c", Shellwords.escape(options["csv_rows"])) if options["csv_rows"] + command.append("-e", Shellwords.escape(options["environment"])) if options["environment"] command.append("-f", Shellwords.escape(options["format"])) if options["format"] + command.append("-r", Shellwords.escape(options["regex"])) if options["regex"] + command.append("-u", Shellwords.escape(options["update_time"])) if options["update_time"] command.append("-m") if options["media_skip"] + # -p may be given with no value (nil, meaning "use last checkpoint") or + # with a regex string. key? distinguishes "not provided" from nil. + if options.key?("proceed") + command.append("-p") + command.append(Shellwords.escape(options["proceed"])) if options["proceed"] + end command.append("-j") if options["json_output"] system(*command) end From 517ef16ebd546b5c9a0ef9b15f436e4633b00c13 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 14:46:24 -0500 Subject: [PATCH 193/222] shorten override path in warning --- lib/datura/python/field_definitions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 36283b1e1..016d706c4 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -312,6 +312,7 @@ def get_fields(omeka_data_base=""): Returns a FieldDefinitions instance (or a CustomFields subclass of it). """ override_path = Path.cwd() / "scripts" / "python" / "field_overrides.py" + override_relative_path = "scripts/python/field_overrides.py" if not override_path.exists(): return FieldDefinitions(omeka_data_base=omeka_data_base) @@ -321,7 +322,7 @@ def get_fields(omeka_data_base=""): spec.loader.exec_module(module) except Exception as e: raise RuntimeError( - f"Failed to load field overrides from {override_path}: {e}" + f"Failed to load field overrides from {override_relative_path}: {e}" ) from e CustomFields = getattr(module, "CustomFields", None) @@ -334,6 +335,6 @@ def get_fields(omeka_data_base=""): # in CustomFields only if you need additional constructor logic. logger.warning( "Field overrides found at %s; custom field mappings will be applied.", - override_path, + override_relative_path, ) return CustomFields(omeka_data_base=omeka_data_base) \ No newline at end of file From 2570a35941a55b6b0f72d85ad449966c6e09d042 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 14:51:59 -0500 Subject: [PATCH 194/222] add c and p to html_and_media_ingest; cleanup --- lib/datura/python/html_and_media_ingest.py | 82 ++++++++++++++++++---- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 14fd1370e..934d83b02 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -44,8 +44,11 @@ OmekaConfigError, OmekaContext, OmekaMediaError, + checkpoint_path, configure_logging, finish_run, + read_checkpoint, + write_checkpoint, ) except ModuleNotFoundError as err: raise SystemExit( @@ -69,8 +72,12 @@ def _parse_args(): Parse command-line arguments for the HTML/media ingest entrypoint. Returns an argparse.Namespace with: + * csv_rows - optional identifier regex for -c item filter, or None * environment - "development" or "production" (default: "development") * format_filter - optional format string for -f (directory-based) filter, or None + * media_skip - bool; True skips items that already have 2+ media objects + * proceed - False (not given), None (-p with no value), or a regex + string (-p "pattern") for checkpoint-based resumption * regex - optional file-filter pattern string, or None * update_time - optional date/time string for -u filter, or None * media_skip - bool; True skips items that already have 2+ media objects @@ -79,6 +86,15 @@ def _parse_args(): parser = argparse.ArgumentParser( description="Attach HTML and IIIF thumbnail media to existing Omeka S items." ) + parser.add_argument( + "-c", "--csv-rows", + default=None, + dest="csv_rows", + help=( + "Only process items whose identifier matches this regex. " + "Mirrors the Ruby -c flag, which filters CSV rows by identifier." + ), + ) parser.add_argument( "-e", "--environment", default="development", @@ -90,6 +106,29 @@ def _parse_args(): dest="format_filter", help="Only post files of this format (tei, csv, vra, ead, html, pdf, webs).", ) + parser.add_argument( + "-m", "--media-skip", + action="store_true", + dest="media_skip", + help=( + "Skip re-ingesting media for items that already have 2 or more " + "media objects (thumbnail + HTML). Useful when re-running the " + "script after a partial failure to avoid re-uploading media that " + "was already successfully ingested." + ), + ) + parser.add_argument( + "-p", "--proceed", + nargs="?", + default=False, + const=None, + dest="proceed", + help=( + "Proceed with media ingest from (and including) the JSON file " + "matching this regex. If given without a value, resumes from " + "the last checkpoint saved in logs/proceed_omeka_html_{environment}." + ), + ) parser.add_argument( "-r", "--regex", default=None, @@ -109,17 +148,6 @@ def _parse_args(): "date-time (2015-01-01T18:24)." ), ) - parser.add_argument( - "-m", "--media-skip", - action="store_true", - dest="media_skip", - help=( - "Skip re-ingesting media for items that already have 2 or more " - "media objects (thumbnail + HTML). Useful when re-running the " - "script after a partial failure to avoid re-uploading media that " - "was already successfully ingested." - ), - ) parser.add_argument( "--log-level", default="INFO", @@ -414,7 +442,9 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): rel = path.relative_to(Path.cwd()) with open(filename) as jsonfile: json_items = json.load(jsonfile) - + # Apply --csv-rows identifier filter if provided. + if ctx.csv_rows: + json_items = omeka.filter_items_by_identifier(ctx.csv_rows, json_items) for json_item in json_items: identifier = json_item.get("identifier") if not identifier: @@ -488,9 +518,10 @@ def main(): 2. Configure root logger. 3. Build OmekaContext (loads config, validates keys, creates API client). 4. Resolve output directories for the requested environment. - 5. Discover JSON files; apply regex filter if -r was passed. - 6. Run media ingest for all items. - 7. Report errors; exit 1 if any failures, 0 if clean. + 5. Discover JSON files; apply csv (-c), format (-f), regex (-r), and update-time (-u) filters. + 6. Apply proceed filter (-p): resume from a checkpoint or a named file. + 7. Run media ingest for all items; writes a checkpoint after each JSON file. + 8. Report errors; exit 1 if any failures, 0 if clean. """ args = _parse_args() start_time = time.time() @@ -516,6 +547,27 @@ def main(): if ctx.update_time: pathlist = filter_items_by_date(ctx.update_time, pathlist) + # Apply -p / --proceed: resume from a specific file or the last checkpoint. + # args.proceed is False (not given), None (-p with no value), or a string. + proceed = args.proceed + if proceed is None: + # -p given with no value — prompt to resume from the saved checkpoint. + last = read_checkpoint(ctx, "omeka_html") + if last is None: + print( + "ERROR: --proceed given with no value but no checkpoint file " + "found at {}.".format(checkpoint_path(ctx, "omeka_html")) + ) + sys.exit(1) + response = input("Continue from {}? (y/n): ".format(last)).strip().lower() + if response == "y": + proceed = last + else: + print("Exiting.") + sys.exit(0) + if proceed: + pathlist = omeka.proceed_files(proceed, pathlist) + logger.info( "Found %d JSON file(s) in %s (environment=%r, media_skip=%s)", len(pathlist), From f28966150618eb8d4d1f71351d338a1dce88db56 Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 14:57:36 -0500 Subject: [PATCH 195/222] add c and p options; cleanup --- lib/datura/python/json_to_omeka.py | 73 ++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 8bbd7ae93..ac5c2579a 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -44,8 +44,11 @@ OmekaAPIError, OmekaConfigError, OmekaContext, + checkpoint_path, configure_logging, finish_run, + read_checkpoint, + write_checkpoint, ) from omeka import filter_items, filter_items_by_date, filter_items_by_format, prepare_item_payload_using_template except ModuleNotFoundError as err: @@ -74,6 +77,9 @@ def _parse_args(): * format_filter - optional format string for -f (directory-based) filter, or None * json_output - bool; True skips post to Omeka API and instead writes data to files as JSON * regex - optional file-filter pattern string, or None + * csv_rows - optional identifier regex for -c item filter, or None + * proceed - False (not given), None (-p with no value), or a regex + string (-p "pattern") for checkpoint-based resumption * update_time - optional date/time string for -u filter, or None * log_level - logging level string, default "INFO" @@ -81,6 +87,15 @@ def _parse_args(): parser = argparse.ArgumentParser( description="Post Datura ES JSON output to an Omeka S instance." ) + parser.add_argument( + "-c", "--csv-rows", + default=None, + dest="csv_rows", + help=( + "Only process items whose identifier matches this regex. " + "Mirrors the Ruby -c flag, which filters CSV rows by identifier." + ), + ) parser.add_argument( "-e", "--environment", default="development", @@ -105,6 +120,18 @@ def _parse_args(): "Omeka item IDs are available." ), ) + parser.add_argument( + "-p", "--proceed", + nargs="?", + default=False, + const=None, + dest="proceed", + help=( + "Proceed with posting from (and including) the JSON file matching " + "this regex. If given without a value, resumes from the last " + "checkpoint saved in logs/proceed_omeka_{environment}." + ), + ) parser.add_argument( "-r", "--regex", default=None, @@ -165,7 +192,9 @@ def post_items(ctx, pathlist, json_output_dir=None): rel = path.relative_to(Path.cwd()) with open(filename) as jsonfile: json_items = json.load(jsonfile) - + # Apply --csv-rows identifier filter if provided. + if ctx.csv_rows: + json_items = omeka.filter_items_by_identifier(ctx.csv_rows, json_items) # template_number is stable for the lifetime of a run — read from # ctx rather than re-reading from config for every item. template_number = ctx.template_number @@ -232,6 +261,10 @@ def post_items(ctx, pathlist, json_output_dir=None): identifier, ) + # Record the last-processed file so that -p (no value) can resume + # from this point on the next run. + write_checkpoint(path.stem, ctx, "omeka") + def add_new_item(ctx, json_item, template_number): """ @@ -344,6 +377,8 @@ def link_items(ctx, pathlist): rel = path.relative_to(Path.cwd()) with open(filename) as jsonfile: json_items = json.load(jsonfile) + if ctx.csv_rows: + json_items = omeka.filter_items_by_identifier(ctx.csv_rows, json_items) for json_item in json_items: identifier = json_item.get("identifier") @@ -446,13 +481,14 @@ def main(): initialises the authenticated API client. Exits with a descriptive error message if the config is missing or malformed (OmekaConfigError). 4. Discover JSON files under output//es/. - 5. Apply regex filter if -r was passed. - 6. Run pass 1 (post_items). - 7. Reset the API client between passes for a clean connection (see - ctx.reset_client() docstring for why this is required). - 8. Run pass 2 (link_items). - 9. Print run summary; exit 1 if any per-item errors were recorded, - 0 if all items succeeded. + 5. Apply csv (-c), format (-f), regex (-r), and update-time (-u) filters. + 6. Apply proceed filter (-p): resume from a checkpoint or a named file. + 7. Add scripts/python to sys.path so that collection-specific + overrides can be imported by field_definitions.get_fields(). + 8. Run pass 1 (post_items); writes a checkpoint after each JSON file. + 9. Reset the API client between passes for a clean connection. + 10. Run pass 2 (link_items); always processes the full filtered pathlist. + 11. Print run summary; exit 1 if any per-item errors were recorded, """ args = _parse_args() start_time = time.time() @@ -481,6 +517,27 @@ def main(): if ctx.update_time: pathlist = filter_items_by_date(ctx.update_time, pathlist) + # Apply -p / --proceed: resume from a specific file or the last checkpoint. + # args.proceed is False (not given), None (-p with no value), or a string. + proceed = args.proceed + if proceed is None: + # -p given with no value — prompt to resume from the saved checkpoint. + last = read_checkpoint(ctx, "omeka") + if last is None: + print( + "ERROR: --proceed given with no value but no checkpoint file " + "found at {}.".format(checkpoint_path(ctx, "omeka")) + ) + sys.exit(1) + response = input("Continue from {}? (y/n): ".format(last)).strip().lower() + if response == "y": + proceed = last + else: + print("Exiting.") + sys.exit(0) + if proceed: + pathlist = omeka.proceed_files(proceed, pathlist) + logger.info( "Found %d JSON file(s) in %s (environment=%r)", len(pathlist), From 0de2e0516bd2f710677b5af3b1b67621a051d7bf Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 15:02:33 -0500 Subject: [PATCH 196/222] add c and p options and helper functions to context; cleanup --- lib/datura/python/omeka_context.py | 72 ++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index 57fb36e20..f553387c2 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -227,6 +227,7 @@ def from_args(cls, args): filter, or None .regex str optional file-filter pattern, or None .update_time str optional date/time string for -u filter, or None + .csv_rows str optional identifier regex for -c filter, or None .media_skip bool skip re-ingesting existing media (html_and_media_ingest only; absent on json_to_omeka args, defaults to False) @@ -261,6 +262,7 @@ def from_args(cls, args): media_skip=getattr(args, "media_skip", False), update_time=parse_update_time(raw_update) if raw_update else None, format_filter=getattr(args, "format_filter", None), + csv_rows=getattr(args, "csv_rows", None), ) @staticmethod @@ -300,7 +302,7 @@ def _load_config(path, env): return contents[env] - def __init__(self, env_config, environment, regex, media_skip, update_time=None, format_filter=None): + def __init__(self, env_config, environment, regex, media_skip, update_time=None, format_filter=None, csv_rows=None): """ Initialise the context. Prefer OmekaContext.from_args() over calling this constructor directly except in tests. @@ -321,6 +323,9 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, * update_time - optional datetime; if set, only items whose source file mtime >= this value are processed (mirrors the -u flag from the main Datura post command) + * csv_rows - optional regex string; if set, only items whose + "identifier" field matches are processed (mirrors the + -c flag from the main Datura post command) """ # ---- Validate required config keys -------------------------------- # Validate up front so that failures are immediate and descriptive. @@ -356,6 +361,7 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, self.media_skip = media_skip self.update_time = update_time self.format_filter = format_filter + self.csv_rows = csv_rows # ---- Config values ------------------------------------------------ self.template_number = env_config["resource_template"] @@ -422,6 +428,7 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, self._fn_build_thumbnail_url = None _process_override_path = Path.cwd() / "scripts" / "python" / "process_overrides.py" + _process_override_relative_path = "scripts/python/process_overrides.py" if _process_override_path.exists(): try: _spec = importlib.util.spec_from_file_location( @@ -444,12 +451,12 @@ def __init__(self, env_config, environment, regex, media_skip, update_time=None, if _active: logger.warning( "Process overrides found at %s; active overrides: %s", - _process_override_path, + _process_override_relative_path, _active, ) except Exception as e: raise OmekaConfigError( - f"Failed to load process overrides from {_process_override_path}: {e}" + f"Failed to load process overrides from {_process_override_relative_path}: {e}" ) from e @@ -609,4 +616,61 @@ def finish_run(ctx, args, start_time): hours, rem = divmod(elapsed, 3600) mins, secs = divmod(rem, 60) print(f"{CYAN}Script finished in {hours:02d} hrs {mins:02d} mins {secs:02d} secs{RESET}") - sys.exit(1 if ctx._errors else 0) \ No newline at end of file + sys.exit(1 if ctx._errors else 0) + +# --------------------------------------------------------------------------- +# Checkpoint helpers (-p / --proceed support) +# --------------------------------------------------------------------------- + +def checkpoint_path(ctx, label): + """ + Return the Path to the checkpoint file for this environment and pipeline. + + The checkpoint file records the stem of the last JSON file that was + successfully processed so that a subsequent run with -p (no value) can + resume from the same point rather than restarting from the beginning. + + Parameters: + * ctx - OmekaContext providing the current environment string + * label - pipeline label used to keep each script's checkpoint separate, + e.g. "omeka" for json_to_omeka or "omeka_html" for + html_and_media_ingest + """ + return Path.cwd() / "logs" / f"proceed_{label}_{ctx.environment}" + + +def read_checkpoint(ctx, label): + """ + Read the last-saved checkpoint stem for this pipeline and environment. + + Returns the identifier stem string written by the most recent + write_checkpoint() call, or None if no checkpoint file exists or the + file is empty (e.g. first run, or file was manually cleared). + + Parameters: + * ctx - OmekaContext + * label - pipeline label (see checkpoint_path) + """ + path = checkpoint_path(ctx, label) + if not path.exists(): + return None + content = path.read_text().strip() + return content if content else None + + +def write_checkpoint(stem, ctx, label): + """ + Save stem as the most recently processed item identifier for this pipeline. + + Called after each JSON file is processed so that a run interrupted + mid-way can be resumed with -p. Creates the logs/ directory if it does + not yet exist. + + Parameters: + * stem - identifier string (JSON file stem) to record, e.g. "abc123" + * ctx - OmekaContext + * label - pipeline label (see checkpoint_path) + """ + path = checkpoint_path(ctx, label) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{stem}\n") \ No newline at end of file From 4121b9637e45e253a3821f8c1987d0da44c1543d Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 15:05:40 -0500 Subject: [PATCH 197/222] add c and p options to omeka.py --- lib/datura/python/omeka.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/lib/datura/python/omeka.py b/lib/datura/python/omeka.py index 4aabb5654..78bcbe596 100644 --- a/lib/datura/python/omeka.py +++ b/lib/datura/python/omeka.py @@ -166,3 +166,57 @@ def filter_items_by_format(format_type, pathlist): # Build the set of stems once source_stems = {sf.stem for sf in source_dir.glob("*")} return [p for p in pathlist if Path(p).stem in source_stems] + + +def filter_items_by_identifier(csv_rows, json_items): + """ + Filter a list of JSON item dicts to those whose identifier matches a regex. + + Used by both entrypoint scripts when -c / --csv-rows is passed on the + command line. Mirrors the Ruby FileCsv row filter: the regex is matched + against the item's "identifier" field (equivalent to the CSV id/identifier + column). + + Parameters: + * csv_rows - regex pattern string, compiled with re.compile() + * json_items - list of item dicts loaded from a Datura ES JSON file + + Returns a (possibly shorter) list containing only the items whose + "identifier" value matches the pattern. + """ + pat = re.compile(csv_rows) + return [item for item in json_items if pat.search(item.get("identifier") or "")] + + +def proceed_files(regex, pathlist): + """ + Filter a sorted list of file paths to those from the first match onward. + + Used by both entrypoint scripts when -p / --proceed is given with a value. + Mirrors the Ruby Helpers.proceed_files() behavior: sorts the list by stem, + finds exactly one matching file, and returns every file from that point to + the end of the list. + + Parameters: + * regex - regex pattern string to locate the starting file by stem + * pathlist - iterable of pathlib.Path objects to filter + + Exits with a descriptive error message if the regex matches zero files + (typo or stale checkpoint) or more than one file (ambiguous — refine the + regex so it matches exactly one starting point). + """ + sorted_paths = sorted(pathlist, key=lambda p: p.stem) + pat = re.compile(regex) + matches = [p for p in sorted_paths if pat.search(p.stem)] + if not matches: + print(f"ERROR: --proceed regex '{regex}' matched no files. Exiting.") + sys.exit(1) + if len(matches) > 1: + names = ", ".join(p.stem for p in matches) + print( + f"ERROR: --proceed regex '{regex}' matched {len(matches)} files ({names}). " + "Refine your regex to match exactly one file. Exiting." + ) + sys.exit(1) + idx = sorted_paths.index(matches[0]) + return sorted_paths[idx:] \ No newline at end of file From c513d75419ee4188a29fd8c67e4ed31e5b41464a Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 26 Jun 2026 16:03:11 -0500 Subject: [PATCH 198/222] limit -c format to csv for extra backstop layer --- bin/post_omeka | 4 ++++ bin/post_omeka_html | 4 ++++ lib/datura/data_manager.rb | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/bin/post_omeka b/bin/post_omeka index ed226b109..69295329d 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -99,6 +99,10 @@ if options.key?("proceed") end ARGV.delete("-j") ARGV.delete("--json-output") +#if -c/--csv-rows is set and no format was specified, restrict generation to CSV only +if options["csv_rows"] && !options["format"] && !ARGV.any? { |a| a == "-f" || a == "--format" || a.start_with?("--format=") } + ARGV.push("-f", "csv") +end #add options to output a json file instead of posting it to Elasticsearch ARGV.unshift("-x", "es", "-o", "-t") #create DataManager before conditional run diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 2ea697c61..039e24143 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -95,6 +95,10 @@ if options.key?("proceed") ARGV.push("--proceed=#{proceed_value}") options["proceed"] = proceed_value end +#if -c/--csv-rows is set and no format was specified, restrict generation to CSV only +if options["csv_rows"] && !options["format"] && !ARGV.any? { |a| a == "-f" || a == "--format" || a.start_with?("--format=") } + ARGV.push("-f", "csv") +end #add option to generate html ARGV.unshift("-x", "html") #create DataManager before conditional run diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index c3eb1745c..34894fe3a 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -211,6 +211,10 @@ def get_files formats = [] if @options["format"] formats = [@options["format"]] + elsif @options["csv_rows"] + msg = "csv_rows filter set (-c); restricting processing to CSV format only" + puts msg.cyan + formats = ["csv"] else formats = Datura::DataManager.format_to_class.keys end From 107ab5b961cb8fd0815d30f59ae1732a46d5fbef Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 9 Jul 2026 14:23:59 -0500 Subject: [PATCH 199/222] revise rescue to raise ArgumentError rather than proceed if csv regex is invalid --- lib/datura/file_types/file_csv.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index b271ac704..e2c3ee0c9 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -155,9 +155,7 @@ def build_csv_row_filter begin Regexp.new(@options["csv_rows"]) rescue RegexpError => e - puts "Warning: --csv-rows value '#{@options["csv_rows"]}' is not a valid regex: #{e.message}".red - puts "Proceeding without row filter — all rows will be processed.".yellow - nil + raise ArgumentError, "Invalid regex '#{options["csv_rows"]}': #{e.message}" end end From d182a6148d70d3bfcbd1349a744de7525201b917 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 13 Jul 2026 10:11:02 -0500 Subject: [PATCH 200/222] add missing write to checkpoint file for omeka html and media --- lib/datura/python/html_and_media_ingest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 934d83b02..42ccf9dcc 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -503,7 +503,9 @@ def process_items(ctx, pathlist, html_dir, iiif_dir): # --- Media pipeline --- ingest_item_media(ctx, json_item, matching_item, html_dir, iiif_dir) - + # Record the last-processed file so that -p (no value) can resume + # from this point on the next run. + write_checkpoint(path.stem, ctx, "omeka_html") # --------------------------------------------------------------------------- # Entrypoint From 778ef69b51f3455f3b3f9239869475be1833572d Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 13 Jul 2026 10:32:42 -0500 Subject: [PATCH 201/222] add try/except blocks for -j in omeka post --- lib/datura/python/json_to_omeka.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index ac5c2579a..49ab703d3 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -214,11 +214,19 @@ def post_items(ctx, pathlist, json_output_dir=None): if json_output_dir is not None: # JSON output mode: build the payload and write it to disk # rather than to the Omeka API. - new_item = api_fields.prepare_item(ctx, json_item) + try: + new_item = api_fields.prepare_item(ctx, json_item) + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "prepare_item", err)) + continue if not new_item: logger.warning("Could not prepare payload for %r; skipping", identifier) continue - payload = prepare_item_payload_using_template(ctx, new_item, template_number) + try: + payload = prepare_item_payload_using_template(ctx, new_item, template_number) + except Exception as err: + ctx.record_error(OmekaAPIError(identifier, "prepare_template_payload", err)) + continue out_path = json_output_dir / f"{identifier}.json" relative_path = f"output/{ctx.environment}/{identifier}.json" logger.info("Writing Omeka payload for %r to %s", identifier, relative_path) From d43c808690c7a62f8e44683a33c38826e4766785 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 13 Jul 2026 10:53:44 -0500 Subject: [PATCH 202/222] add regex validation to omeka scripts (Ruby helpers check with Python backstop) --- bin/post_omeka | 2 ++ bin/post_omeka_html | 2 ++ lib/datura/python/html_and_media_ingest.py | 7 +++++++ lib/datura/python/json_to_omeka.py | 7 +++++++ lib/datura/python/omeka_context.py | 23 ++++++++++++++++++++++ 5 files changed, 41 insertions(+) diff --git a/bin/post_omeka b/bin/post_omeka index 69295329d..da5847dff 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -59,6 +59,8 @@ optparse = OptionParser.new do |opts| end #parse, but do not consume, command line arguments (the usual parse! would consume them) optparse.parse(ARGV) +Datura::Helpers.validate_regex(options["regex"], "--regex") if options["regex"] +Datura::Helpers.validate_regex(options["csv_rows"], "--csv-rows") if options["csv_rows"] #remove options not used in the main script ARGV.delete("-s") ARGV.delete("--skip") diff --git a/bin/post_omeka_html b/bin/post_omeka_html index 039e24143..e01dbd6cd 100755 --- a/bin/post_omeka_html +++ b/bin/post_omeka_html @@ -55,6 +55,8 @@ optparse = OptionParser.new do |opts| end #parse, but do not consume, command line arguments (the usual parse! would consume them) optparse.parse(ARGV) +Datura::Helpers.validate_regex(options["regex"], "--regex") if options["regex"] +Datura::Helpers.validate_regex(options["csv_rows"], "--csv-rows") if options["csv_rows"] #remove options not used in the main script ARGV.delete("-m") ARGV.delete("--media_skip") diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index 42ccf9dcc..a05ab3cd5 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -48,6 +48,7 @@ configure_logging, finish_run, read_checkpoint, + validate_regex_arg, write_checkpoint, ) except ModuleNotFoundError as err: @@ -529,6 +530,12 @@ def main(): start_time = time.time() configure_logging(args.log_level) + # Validate regex for -r and -c option input + if args.regex: + validate_regex_arg(args.regex, "--regex") + if args.csv_rows: + validate_regex_arg(args.csv_rows, "--csv-rows") + # OmekaConfigError propagates here as a fatal error — missing or broken # config means no API access is possible. ctx = OmekaContext.from_args(args) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 49ab703d3..1031735e5 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -48,6 +48,7 @@ configure_logging, finish_run, read_checkpoint, + validate_regex_arg, write_checkpoint, ) from omeka import filter_items, filter_items_by_date, filter_items_by_format, prepare_item_payload_using_template @@ -505,6 +506,12 @@ def main(): # errors are captured at the correct level. configure_logging(args.log_level) + # Validate regex for -r and -c option input + if args.regex: + validate_regex_arg(args.regex, "--regex") + if args.csv_rows: + validate_regex_arg(args.csv_rows, "--csv-rows") + # OmekaContext.from_args() raises OmekaConfigError (a subclass of # OmekaError) if config/private.yml is missing, unparseable, or missing # a required key. Let this propagate to the top level — configuration diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index f553387c2..b1a56c988 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -9,6 +9,7 @@ import logging from logging.handlers import RotatingFileHandler import os +import re import sys from field_definitions import get_fields @@ -189,6 +190,28 @@ def parse_update_time(s): f"Expected 'today', a date (2015-01-01), or date-time (2015-01-01T18:24).{RESET}" ) +# --------------------------------------------------------------------------- +# Regex parsing +# --------------------------------------------------------------------------- + +def validate_regex_arg(pattern, flag): + """ + Compile a regex string, raising OmekaConfigError immediately if invalid. + + Mirrors the Ruby Datura::Helpers.validate_regex check. Called in each + entrypoint's main() before OmekaContext is built, so that an invalid + -r or -c value exits with a clean error message rather than a traceback. + + Parameters: + * pattern - the regex string from the CLI argument + * flag - the flag name for the error message, e.g. "--regex" + """ + try: + re.compile(pattern) + except re.error as e: + raise OmekaConfigError( + f"Invalid regex for {flag} {pattern!r}: {e}" + ) # --------------------------------------------------------------------------- # Context From da0f0bd96a03413b627a6beddde11cce0e24df71 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 13 Jul 2026 14:55:58 -0500 Subject: [PATCH 203/222] delete stale json before full omeka post --- bin/post_omeka | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bin/post_omeka b/bin/post_omeka index da5847dff..5dd448014 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -109,6 +109,20 @@ end ARGV.unshift("-x", "es", "-o", "-t") #create DataManager before conditional run manager = Datura::DataManager.new +# On full runs (no limiting flags), clear all ES output before transforms so +# that deleted source files do not persist in the output directory. This makes +# the output directory a faithful reflection of the current source tree. +# -s (skip transforms) is supported for limited runs only; combining -s with +# an unrestricted full run would clear the output and leave nothing to post. +is_limited_run = options["regex"] || options["format"] || options["proceed"] || options["csv_rows"] +if generate_es && !is_limited_run + env = options.fetch("environment", "development") + out_es = File.join(Dir.pwd, "output", env, "es") + if Dir.exist?(out_es) + cleared = Dir.glob("#{out_es}/*.json").count { |f| File.delete(f) } + puts "Cleared #{cleared} file(s) from output/#{env}/es" if cleared > 0 + end +end if generate_es manager.run end From 3f0b14b19e7df7b3e9f1e0bc2130d60031cfcbba Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 14 Jul 2026 10:30:29 -0500 Subject: [PATCH 204/222] update omeka overrides paths and setup docs --- README.md | 7 ++++--- bin/setup | 3 ++- docs/1_setup/omeka_setup.md | 5 ++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5177e65f2..c6adb8183 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ source "https://rubygems.org" gem "datura", git: "https://github.com/CDRH/datura.git", tag: "v0.0.0" ``` -If this is the first datura repository on your machine, you will need to install Python 3 and `saxonche`. `saxonche` is included in `requirements.txt`, so you can install it by following the Omeka setup [instructions](/docs/1_setup/omeka_setup.md#enabling-a-virtual-environment) for installing Python dependencies. - After that, in the directory with the Gemfile, run the following: ``` @@ -29,7 +27,10 @@ bundle install bundle exec setup ``` -The last step should add files and some basic directories. Have a look at the [setup instructions](/docs/1_setup/collection_setup.md) to learn how to add your files and start working with the data! +The setup step should add files and some basic directories. If this is the first datura repository on your machine, you will need to install Python 3. You will also need to install a few `saxonche`. `saxonche` is included in `requirements.txt`, which should have been added as part of setup, so you can install it by following the Omeka setup [instructions](/docs/1_setup/omeka_setup.md#enabling-a-virtual-environment) for installing Python dependencies. + +Now you are ready to have a look at the [setup instructions](/docs/1_setup/collection_setup.md) to learn how to add your files and start working with the data! + ### RVM diff --git a/bin/setup b/bin/setup index 383c9b901..82bcc5d49 100755 --- a/bin/setup +++ b/bin/setup @@ -98,7 +98,8 @@ FileUtils.touch(File.join(coll, "scripts", "overrides", ".keep")) puts "-- Place Python overrides in scripts/python" FileUtils.mkdir_p(File.join(coll, "scripts", "python")) -FileUtils.cp(File.join(datura, "lib", "datura", "python", "omeka_overrides_example.py"), File.join(coll, "scripts", "python", "omeka_overrides_example.py")) +FileUtils.cp(File.join(datura, "lib", "datura", "python", "field_overrides_example.py"), File.join(coll, "scripts", "python", "field_overrides_example.py")) +FileUtils.cp(File.join(datura, "lib", "datura", "python", "process_overrides_example.py"), File.join(coll, "scripts", "python", "process_overrides_example.py")) puts "-- Copying requirements.txt for Python/Omeka pipeline".green FileUtils.cp(File.join(datura, "requirements.txt"), File.join(coll, "requirements.txt")) diff --git a/docs/1_setup/omeka_setup.md b/docs/1_setup/omeka_setup.md index 7db5ca967..b698ce61c 100644 --- a/docs/1_setup/omeka_setup.md +++ b/docs/1_setup/omeka_setup.md @@ -4,7 +4,7 @@ #### If you would like to create a new repository -Follow [the steps](https://github.com/CDRH/datura/blob/dev/docs/1_setup/collection_setup.md#step-1--create-a-new-collection-directory) in the `collection_setup` documentation for Datura, specifying any recent release of Datura in your Gemfile. If you know out of the gate that you plan to create overrides for any Omeka fields, copy `omeka_overrides_examples.py` (in the `/scripts/python` directory) to `omeka_overrides.py`. +Follow [the steps](https://github.com/CDRH/datura/blob/dev/docs/1_setup/collection_setup.md#step-1--create-a-new-collection-directory) in the `collection_setup` documentation for Datura, specifying any recent release of Datura in your Gemfile. If you know out of the gate that you plan to create overrides for any Omeka processes or fields, copy `process_overrides_examples.py` and/or `field_overrides_examples.py` (in the `/scripts/python` directory) to `process_overrides.py` or `field_overrides.py`. #### If you are working with an existing data repository @@ -40,11 +40,10 @@ You should now see a `(.venv)` at the front of the command line prompt. You will Next, confirm you have a `requirements.txt` file in the root directory of your collection. If you are working with an existing repository, you may need to copy this over from Datura. Then, to install the dependencies, run: ```bash -pip3 install packaging pip3 install -r requirements.txt ``` -The `packaging` library will need to be installed separately so the `omeka_s_tools` installation (part of the `requirements.txt` list) will install correctly. If installation fails, `pip` may need to be upgraded (the error message should advise this). +The `packaging` library may need to be installed separately if the `omeka_s_tools` installation (part of the `requirements.txt` list) does not install correctly. If installation fails, `pip` may need to be upgraded (the error message should advise this). ### Step 4: Set up config for Omeka S posting From 61f6c60ffbfbfed5e2cd17987d5b0df9b1a7414b Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 14 Jul 2026 10:53:33 -0500 Subject: [PATCH 205/222] add api version to default config --- bin/setup | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/setup b/bin/setup index 82bcc5d49..689968737 100755 --- a/bin/setup +++ b/bin/setup @@ -25,6 +25,7 @@ File.open(File.join(coll, "config", "private.yml"), "w") do |file| default: # number of files processed at the same time, increase for more powerful computers threads: 5 + api_version: "2.0" # Omeka S posting (required only if using post_omeka / post_omeka_html) # omeka_server: servername.unl.edu/path/to/api From 992ce5db5350881204ec030ab8590cbb6fb3412e Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 14 Jul 2026 11:19:45 -0500 Subject: [PATCH 206/222] restructure requirements, update docs --- docs/1_setup/omeka_setup.md | 2 +- requirements.in | 4 +++ requirements.txt | 63 +++++++++++++++++++++++++++---------- 3 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 requirements.in diff --git a/docs/1_setup/omeka_setup.md b/docs/1_setup/omeka_setup.md index b698ce61c..77cc24877 100644 --- a/docs/1_setup/omeka_setup.md +++ b/docs/1_setup/omeka_setup.md @@ -43,7 +43,7 @@ Next, confirm you have a `requirements.txt` file in the root directory of your c pip3 install -r requirements.txt ``` -The `packaging` library may need to be installed separately if the `omeka_s_tools` installation (part of the `requirements.txt` list) does not install correctly. If installation fails, `pip` may need to be upgraded (the error message should advise this). +The `packaging` and `setuptools` libraries may need to be installed separately if the `omeka_s_tools` installation (part of the `requirements.txt` list) does not install correctly. If installation fails, `pip` may need to be upgraded (the error message should advise this). ### Step 4: Set up config for Omeka S posting diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..4ac4912ab --- /dev/null +++ b/requirements.in @@ -0,0 +1,4 @@ +# requirements.in +omeka_s_tools @ git+https://github.com/CDRH/omeka_s_tools.git@will_changes +pyyaml>=6.0.2,<7.0 +saxonche>=12.9,<13.0 # 12.5.0 wheels unavailable for recent Python/platform updates; 12.9 is latest stable 12.x per Saxonica docs diff --git a/requirements.txt b/requirements.txt index 0f60978c5..33d1b1be7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,47 @@ -attrs==25.3.0 -cattrs==25.1.1 -certifi==2025.8.3 -charset-normalizer==3.4.3 -idna==3.10 -omeka_s_tools @ git+https://github.com/CDRH/omeka_s_tools.git@will_changes -packaging==25.0 -platformdirs==4.4.0 -PyYAML==6.0.2 -requests==2.32.5 -requests-cache==1.2.1 -setuptools==82.0.1 -saxonche==12.5.0 -typing_extensions==4.15.0 -url-normalize==2.2.1 -urllib3==2.5.0 +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# pip-compile requirements.in +# +attrs==26.1.0 + # via + # cattrs + # requests-cache +cattrs==26.1.0 + # via requests-cache +certifi==2026.6.17 + # via requests +charset-normalizer==3.4.9 + # via requests +idna==3.18 + # via + # requests + # url-normalize +omeka-s-tools @ git+https://github.com/CDRH/omeka_s_tools.git@will_changes + # via -r requirements.in +packaging==26.2 + # via omeka-s-tools +platformdirs==4.10.0 + # via requests-cache +pyyaml==6.0.3 + # via -r requirements.in +requests==2.34.2 + # via + # omeka-s-tools + # requests-cache +requests-cache==1.3.3 + # via omeka-s-tools +saxonche==12.9.0 + # via -r requirements.in +typing-extensions==4.16.0 + # via cattrs +url-normalize==3.0.0 + # via requests-cache +urllib3==2.7.0 + # via + # requests + # requests-cache + +# The following packages are considered to be unsafe in a requirements file: +# pip From ee297c6df89776020ba2b1ae31c06cf09760fb9d Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 14 Jul 2026 12:40:09 -0500 Subject: [PATCH 207/222] update docs to reflect new stale json deletion step --- docs/3_manage/post_omeka.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/3_manage/post_omeka.md b/docs/3_manage/post_omeka.md index e162f527c..a0f9eb13d 100644 --- a/docs/3_manage/post_omeka.md +++ b/docs/3_manage/post_omeka.md @@ -2,15 +2,15 @@ See (omeka setup instructions)[../1_setup/omeka_setup.md] for how to prepare and configure your data repository and activate the Python virtual environment. -Running the `post_omeka` script will first run the Datura scripts to generate JSON files with the standard fields and values of the CDRH API (this is what is normally sent to ElasticSearch when you run `post`). This first step is equivalent to running `post -x es -o -t`. It then sends the generated JSON to the Python scripts to be ingested into Omeka S. +Running the `post_omeka` script will first run the Datura scripts to generate JSON files with the standard fields and values of the CDRH API (this is what is normally sent to ElasticSearch when you run `post`). This first step is equivalent to running `post -x es -o -t`. It then sends the generated JSON to the Python scripts to be ingested into Omeka S. Note that this command also deletes existing content from the output `es` directory so that any files that have been deleted from source do not populate from stale JSON. -Use the `-s` option to skip the generation step and only post to Omeka S (requires that you have already generated the needed documents by running `post_omeka` normally). +NOTE: if and only if you are posting a subset of items, you may use the `-s` option to skip the JSON generation step and only post to Omeka S (requires that you have already generated the needed documents by running `post_omeka` normally). If you run `post_omeka` with `-s` and no filter, the script will delete all JSON output in the `es` directory and will therefore have nothing to post to Omeka. It is possible to run `post_omeka` with Datura's other command line options as described in [post.md] (for instance `-f` to filter by format and `-r` to filter by regex), but it is not recommended to override the default options such as `-x es`. You can specify the environment with `-e [environment]` but you must set an `item_set` with the desired environment in `config/private.yml.` See (omeka setup instructions)[../1_setup/omeka_setup.md] for more details. -For information on how to override field definitions, see [Omeka Overrides](../2_customization/omeka_overrides.md). +For information on how to override processes and field definitions, see [Omeka Overrides](../2_customization/omeka_overrides.md). ## Troubleshooting From 6e27de6d88099929ff7072c538456ca89f25b3f8 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 14 Jul 2026 12:43:38 -0500 Subject: [PATCH 208/222] add cautionary note to -s description in post docs --- docs/3_manage/post.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/3_manage/post.md b/docs/3_manage/post.md index 18954b249..482e31a97 100644 --- a/docs/3_manage/post.md +++ b/docs/3_manage/post.md @@ -125,7 +125,7 @@ Example: `post -r let0001` -s, --skip ``` -Omeka specific, this skips the JSON or HTML generation step and only posts to Omeka S (it can be used with both `post_omeka` and `post_omeka_html`). +Omeka specific, this skips the JSON or HTML generation step and only posts to Omeka S (it can be used with both `post_omeka` and `post_omeka_html`). Note however in the case of `post_omeka` that it should only be used with a filtered subset of files (otherwise nothing will be posted, because JSON files are deleted as part of unfiltered posts). ```bash -t, --transform-only From 921da9db525c759af7e7c28c34902a4ae2c78f1a Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 15 Jul 2026 09:30:09 -0500 Subject: [PATCH 209/222] add rescue to json delete loop --- bin/post_omeka | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/post_omeka b/bin/post_omeka index 5dd448014..23ffacc34 100755 --- a/bin/post_omeka +++ b/bin/post_omeka @@ -119,8 +119,18 @@ if generate_es && !is_limited_run env = options.fetch("environment", "development") out_es = File.join(Dir.pwd, "output", env, "es") if Dir.exist?(out_es) - cleared = Dir.glob("#{out_es}/*.json").count { |f| File.delete(f) } - puts "Cleared #{cleared} file(s) from output/#{env}/es" if cleared > 0 + files = Dir.glob("#{out_es}/*.json") + cleared = 0 + + files.each do |f| + begin + File.delete(f) + cleared += 1 + rescue => e + puts "There was an error deleting a file: #{e.message}".red + end + end + puts "Cleared #{cleared} file(s) from output/#{env}/es" end end if generate_es From 52dab2c6a85feccbdcc73357b294d578f5096b0b Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 16 Jul 2026 13:25:22 -0500 Subject: [PATCH 210/222] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c6adb8183..436fc1f3c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ bundle install bundle exec setup ``` -The setup step should add files and some basic directories. If this is the first datura repository on your machine, you will need to install Python 3. You will also need to install a few `saxonche`. `saxonche` is included in `requirements.txt`, which should have been added as part of setup, so you can install it by following the Omeka setup [instructions](/docs/1_setup/omeka_setup.md#enabling-a-virtual-environment) for installing Python dependencies. +The setup step should add files and some basic directories. If this is the first datura repository on your machine, you will need to install Python 3. You will also need to install `saxonche`. `saxonche` is included in `requirements.txt`, which should have been added as part of setup, so you can install it by following the Omeka setup [instructions](/docs/1_setup/omeka_setup.md#enabling-a-virtual-environment) for installing Python dependencies. Now you are ready to have a look at the [setup instructions](/docs/1_setup/collection_setup.md) to learn how to add your files and start working with the data! From b528b0b3452354f7d60e7c07b73ba06e4be1c69f Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 16 Jul 2026 13:25:40 -0500 Subject: [PATCH 211/222] remove note about pip from requirements --- requirements.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 33d1b1be7..e0616ca8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,6 +42,3 @@ urllib3==2.7.0 # via # requests # requests-cache - -# The following packages are considered to be unsafe in a requirements file: -# pip From 36cfd5e3d95de4a16f9f52e136486e3616791adf Mon Sep 17 00:00:00 2001 From: nichgray Date: Thu, 16 Jul 2026 13:26:13 -0500 Subject: [PATCH 212/222] update overrides docs --- docs/2_customization/omeka_overrides.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/2_customization/omeka_overrides.md b/docs/2_customization/omeka_overrides.md index 24ac84134..01c7d74a5 100644 --- a/docs/2_customization/omeka_overrides.md +++ b/docs/2_customization/omeka_overrides.md @@ -6,7 +6,7 @@ Each Omeka field is updated by the method in [api_fields.py](../../../lib/datura ### Overriding fields -To override the field definitions, copy the file [omeka_overrides_example.py](../../../lib/datura/python/omeka_overrides_example.py) to [omeka_overrides.py](../../../lib/datura/python/omeka_overrides.py) in the `scripts/python` file of the project directory. Then override each method as needed, using the commented patterns in the example overrides file as a guide. +To override the field definitions, copy the file [field_overrides_example.py](../../../lib/datura/python/field_overrides_example.py) to [field_overrides.py](../../../lib/datura/python/field_overrides.py) in the `scripts/python` file of the project directory. Then override each method as needed, using the commented patterns in the example overrides file as a guide. Each overridden method needs to take the arguments `self` (a Python placeholder for a class instance) and `json` (representing the generated JSON) and to match the methods defined in `field_definitions.py`. (The same goes for adding new methods to `field_definitions.py`.) For instance: @@ -22,15 +22,19 @@ For instance: First retrieve the value from the Elasticsearch `json` (keeping in mind that it is sometimes nil), then do any manipulations needed before returning the desired value. The return value must be either an list or single value. For single values, usually this will be the same as the value in the JSON. But unlike the ElasticSearch-based API, it is not possible to ingest nested fields into Omeka S, so they must be reduced into array form. See [field_definitions.py](../../../lib/datura/python/field_definitions.py)for examples of how to retrieve single and nested values from the JSON, manipulate them and return the proper values for Omeka S. +### Overriding processes + +Some core functions involved in the Omeka posting can also be overridden. To do this, copy the file [process_overrides_example.py](../../../lib/datura/python/process_overrides_example.py) to [process_overrides.py](../../../lib/datura/python/process_overrides.py) in the `scripts/python` file of the project directory. Then override each method as needed, using the commented patterns in the example overrides file as a guide. + ### Linking items -Any new fields that link to the id of another item should be added to the `link_item` in [api_fields.py](../../../lib/datura/python/api_fields.py). `link_item_record` works in the same way as `update_item_value` but the value of the ES field must be a CDRH ID. +Any new fields that link to the id of another item should be added to the `link_records` in `process_overrides.py`. `link_item_record` works in the same way as `update_item_value` but the value of the ES field must be a CDRH ID. ```python try: part_ids = [part['id'] for part in json_item["has_part"]] link_item_record(ctx, existing_item, "dcterms:hasPart", part_ids) - except Exception: + except (KeyError, TypeError): pass ``` From be591bca7bac5c195e91b5c5beabe7c611d38856 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 21 Jul 2026 13:44:53 -0500 Subject: [PATCH 213/222] set items for all envs to default private --- lib/datura/python/omeka_context.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/datura/python/omeka_context.py b/lib/datura/python/omeka_context.py index b1a56c988..a8ca24379 100644 --- a/lib/datura/python/omeka_context.py +++ b/lib/datura/python/omeka_context.py @@ -502,13 +502,16 @@ def item_set_id(self): def is_public(self): # type: () -> bool """ - True only when environment is "production". + Always returns False. Items are posted as private (visible only to logged-in Omeka + admins) regardless of environment. Visibility can be changed manually in the Omeka S + admin interface after ingest if needed. + + If it is desired to change this behavior at some future point such that items posted + to the production environment will be public by default, replace `return False` below + with `return self.environment == "production"`. - Items created with is_public=False are visible only to logged-in Omeka - admins, which prevents in-progress development ingests from appearing - to public users of the site. """ - return self.environment == "production" + return False # ----------------------------------------------------------------------- # API helpers From f41d74e8147c03e401f9e64827185aea79b3b52b Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 21 Jul 2026 14:09:37 -0500 Subject: [PATCH 214/222] add option for is_public metadata field override --- lib/datura/python/api_fields.py | 3 +++ lib/datura/python/field_definitions.py | 12 ++++++++++++ lib/datura/python/field_overrides_example.py | 8 +++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/datura/python/api_fields.py b/lib/datura/python/api_fields.py index ee8aed88c..c604983ee 100644 --- a/lib/datura/python/api_fields.py +++ b/lib/datura/python/api_fields.py @@ -202,6 +202,9 @@ def add_formatted_value(ctx, item, key, value, datatype, label=""): } formatted = ctx.client.prepare_property_value(prop_value, prop_id, label) + if key in ctx.fields.private_fields(): + formatted["is_public"] = False + if key in item and isinstance(item[key],list): item[key].append(formatted) else: diff --git a/lib/datura/python/field_definitions.py b/lib/datura/python/field_definitions.py index 016d706c4..34f16ce5e 100644 --- a/lib/datura/python/field_definitions.py +++ b/lib/datura/python/field_definitions.py @@ -30,6 +30,18 @@ def __init__(self, omeka_data_base=""): # Stored as a private attribute and accessed only by uriData(). self._omeka_data_base = omeka_data_base + def private_fields(self): + """ + Return a set of Omeka property terms whose values should be posted with + is_public=False. Override in field_overrides.py to mark specific fields + as private for this collection. + + Example: + def private_fields(self): + return {"dcterms:identifier", "dcterms:source"} + """ + return set() + def field_manifest(self): """ Declare the ordered list of Omeka property mappings for this collection. diff --git a/lib/datura/python/field_overrides_example.py b/lib/datura/python/field_overrides_example.py index 98bc03647..0720328ab 100644 --- a/lib/datura/python/field_overrides_example.py +++ b/lib/datura/python/field_overrides_example.py @@ -37,4 +37,10 @@ class CustomFields(FieldDefinitions): # if date_to_parse and "-01-01" in date_to_parse: # return datetime.strptime(date_to_parse, "%Y-%m-%d").year # else: - # return date_to_parse \ No newline at end of file + # return date_to_parse + + # Pattern 6: mark specific metadata fields as private at the value level + # values for these terms will be posted with is_public: false in the Omeka S API payload, + # hiding them from public view regardless of item-level visibility + # def private_fields(self): + # return {"dcterms:identifier", "dcterms:source"} \ No newline at end of file From 0de8b47517fdb3374a06c32e73abeb91028fdcb7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Tue, 21 Jul 2026 14:43:07 -0500 Subject: [PATCH 215/222] exit immediately if no matching json files are found --- lib/datura/python/json_to_omeka.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index 1031735e5..c8b75d12a 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -560,6 +560,13 @@ def main(): ctx.environment, ) + if not pathlist: + logger.warning( + "No JSON files found in output/%s/es — nothing to post or link. Exiting.", + ctx.environment, + ) + sys.exit(0) + # --- JSON output mode (-j / --json-output) --- if args.json_output: relative_dir = f"output/{ctx.environment}/omeka" From c59824a2f27ea5a98bd3769e7a37369cd33b06a7 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 22 Jul 2026 15:09:15 -0500 Subject: [PATCH 216/222] shift to f-string --- lib/datura/python/json_to_omeka.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/datura/python/json_to_omeka.py b/lib/datura/python/json_to_omeka.py index c8b75d12a..729037539 100644 --- a/lib/datura/python/json_to_omeka.py +++ b/lib/datura/python/json_to_omeka.py @@ -561,10 +561,7 @@ def main(): ) if not pathlist: - logger.warning( - "No JSON files found in output/%s/es — nothing to post or link. Exiting.", - ctx.environment, - ) + logger.warning(f"No JSON files found in output/{ctx.environment}/es — nothing to post or link. Exiting.") sys.exit(0) # --- JSON output mode (-j / --json-output) --- From a3e48809ae1d578b139a4ed67c2577c02742af2f Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 29 Jul 2026 16:13:08 -0500 Subject: [PATCH 217/222] add count and compiled end-run list for skipped es items --- lib/datura/data_manager.rb | 7 ++++++- lib/datura/file_type.rb | 7 +++++-- lib/datura/file_types/file_csv.rb | 5 +++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/datura/data_manager.rb b/lib/datura/data_manager.rb index 34894fe3a..9bcd1754e 100644 --- a/lib/datura/data_manager.rb +++ b/lib/datura/data_manager.rb @@ -11,6 +11,7 @@ class Datura::DataManager attr_accessor :error_html attr_accessor :error_iiif attr_accessor :error_solr + attr_accessor :skipped_es attr_accessor :files attr_accessor :options @@ -37,6 +38,7 @@ def initialize @error_html = [] @error_iiif = [] @error_solr = [] + @skipped_es = [] # combine user input and config files params = Datura::Parser.post_params @@ -173,6 +175,7 @@ def end_run error_msg << "#{@error_html.length} HTML transform error(s)\n" error_msg << "#{@error_iiif.length} IIIF Manifest transform error(s)\n" error_msg << "#{@error_solr.length} Solr transform / post error(s)\n" + error_msg << "#{@skipped_es.length} ES item(s) skipped (missing id or title)\n" puts error_msg @log.info(error_msg) @@ -180,7 +183,8 @@ def end_run "ES" => @error_es, "HTML" => @error_html, "IIIF" => @error_iiif, - "Solr" => @error_solr + "Solr" => @error_solr, + "ES skipped" => @skipped_es }.reject { |_, v| v.empty? } if all_errors.any? @@ -418,6 +422,7 @@ def transform_and_post(file) rescue => e error_with_transform_and_post("#{e}", @error_es) end + @skipped_es.concat(file.skipped_es) if file.skipped_es.any? # html begin diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 817685bd1..88a3cb6a1 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -6,6 +6,7 @@ class FileType # general information about file attr_reader :file_location attr_reader :options + attr_reader :skipped_es # script locations attr_accessor :script_es @@ -22,6 +23,7 @@ class FileType def initialize(location, options) @file_location = location @options = options + @skipped_es = [] add_xsl_params_options # set output directories output = File.join(@options["collection_dir"], "output", @options["environment"]) @@ -68,8 +70,9 @@ def post_es(es) if transformed && transformed.length > 0 transformed.each do |doc| if doc["identifier"].to_s.empty? || doc["title"].to_s.empty? - puts "skipping item without id or title".red - puts "check line ".red + doc.values.join("; ").strip.red + msg = "skipping item without id or title\n check line #{new_row.values.join("; ").strip[0..100]}" + puts msg.yellow + @skipped_es << msg next end id = doc["identifier"] diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index e2c3ee0c9..0e52960d2 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -87,8 +87,9 @@ def transform_es if !row_to_es["identifier"].to_s.empty? && !row_to_es["title"].to_s.empty? es_doc << row_to_es else - puts "skipping item without id or title".red - puts "check line ".red + row.to_s.strip[0..400].red + msg = "skipping item without id or title\n check line #{new_row.values.join("; ").strip[0..100]}" + puts msg.yellow + @skipped_es << msg next end end From f6e3110b966314b99acdf7f0d5befe5038e64ade Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 5 Aug 2026 11:31:13 -0500 Subject: [PATCH 218/222] fix msg syntax for skipped items --- lib/datura/file_type.rb | 2 +- lib/datura/file_types/file_csv.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datura/file_type.rb b/lib/datura/file_type.rb index 88a3cb6a1..a88ed496c 100644 --- a/lib/datura/file_type.rb +++ b/lib/datura/file_type.rb @@ -70,7 +70,7 @@ def post_es(es) if transformed && transformed.length > 0 transformed.each do |doc| if doc["identifier"].to_s.empty? || doc["title"].to_s.empty? - msg = "skipping item without id or title\n check line #{new_row.values.join("; ").strip[0..100]}" + msg = "Skipping item without id or title: #{doc.values.join('; ').strip[0..100]}" puts msg.yellow @skipped_es << msg next diff --git a/lib/datura/file_types/file_csv.rb b/lib/datura/file_types/file_csv.rb index 0e52960d2..532c26e3e 100644 --- a/lib/datura/file_types/file_csv.rb +++ b/lib/datura/file_types/file_csv.rb @@ -87,7 +87,7 @@ def transform_es if !row_to_es["identifier"].to_s.empty? && !row_to_es["title"].to_s.empty? es_doc << row_to_es else - msg = "skipping item without id or title\n check line #{new_row.values.join("; ").strip[0..100]}" + msg = "Skipping item without id or title: check line #{row.to_s.strip[0..200]}" puts msg.yellow @skipped_es << msg next From 7dd288ee2e2d2d29b60edd50fdf932205b620448 Mon Sep 17 00:00:00 2001 From: nichgray Date: Wed, 12 Aug 2026 15:45:36 -0500 Subject: [PATCH 219/222] adjust media is_public to True rather than pulling from context --- docs/3_manage/post_omeka_html.md | 8 ++++---- lib/datura/python/html_and_media_ingest.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/3_manage/post_omeka_html.md b/docs/3_manage/post_omeka_html.md index 1021e6359..dd54c9656 100644 --- a/docs/3_manage/post_omeka_html.md +++ b/docs/3_manage/post_omeka_html.md @@ -14,12 +14,12 @@ For information on how to override field definitions, see [Omeka Overrides](../2 ### Media ingesters (for developers) -The media payload, set in `html_and_media_ingest.py`, must be structured in a specific way to add items. It is different in the case of html and iiif images. The `is_public` field is set in `omeka_context.py` and is based on environment (`production = True`, everything else = `False`). +The media payload, set in `html_and_media_ingest.py`, must be structured in a specific way to add items. It is different in the case of html and iiif images. For an html field: ```json { - "o:is_public": ctx.is_public, + "o:is_public": True, "data": { "html": html_content, }, @@ -29,7 +29,7 @@ For an html field: For a file upload (i.e. to upload): ```json { - "o:is_public": ctx.is_public, + "o:is_public": True, "data": { "upload": html_content, }, @@ -39,7 +39,7 @@ For a file upload (i.e. to upload): For posting to the IIIF ingester (not currently implemented): ```json { - "o:is_public": ctx.is_public, + "o:is_public": True, "data": { "upload": iiif_url }, diff --git a/lib/datura/python/html_and_media_ingest.py b/lib/datura/python/html_and_media_ingest.py index a05ab3cd5..554b10274 100644 --- a/lib/datura/python/html_and_media_ingest.py +++ b/lib/datura/python/html_and_media_ingest.py @@ -305,7 +305,7 @@ def ingest_thumbnail(ctx, json_item, matching_item, iiif_dir): # the same term do not make redundant API requests. try: media_payload = { - "o:is_public": ctx.is_public, + "o:is_public": True, "data": { "upload": str(thumbnail_local), "dcterms:title": ctx.client.prepare_property_value( @@ -375,7 +375,7 @@ def ingest_html(ctx, json_item, matching_item, html_dir): return media_payload = { - "o:is_public": ctx.is_public, + "o:is_public": True, "data": { "html": html_content, }, From 056fef21d239c6f3981f3645aab636bbf52e35df Mon Sep 17 00:00:00 2001 From: nichgray Date: Fri, 14 Aug 2026 13:57:50 -0500 Subject: [PATCH 220/222] add full as fallback for iiif path when no param is listed --- lib/xslt/tei_to_html/lib/formatting.xsl | 28 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/xslt/tei_to_html/lib/formatting.xsl b/lib/xslt/tei_to_html/lib/formatting.xsl index f26b408be..96dd1b4c5 100644 --- a/lib/xslt/tei_to_html/lib/formatting.xsl +++ b/lib/xslt/tei_to_html/lib/formatting.xsl @@ -558,10 +558,16 @@ %2F - .jpg/full/! - - , - + .jpg/full/ + + + ! + + , + + + full + /0/default.jpg @@ -577,10 +583,16 @@ %252F - .jpg/full/! - - , - + .jpg/full/ + + + ! + + , + + + full + /0/default.jpg From c76e63e6c1f82ae13a6a471f149ba249daf043ee Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 17 Aug 2026 14:15:27 -0500 Subject: [PATCH 221/222] update changelog for v2.0.0 --- CHANGELOG.md | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0973b094f..01ffab105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,22 +25,39 @@ Versioning](https://semver.org/spec/v2.0.0.html). ### Security --> -## [Unreleased] - Brief description TBD before next release -[Unreleased]: https://github.com/CDRH/datura/compare/v1.1.0...dev - -### Fixed +## [v2.0.0] - 2026-08-20 - Shift to saxonche, improve error handling, refactor Omeka S posting, add post options +[v2.0.0]: https://github.com/CDRH/datura/compare/v1.1.0...v2.0.0 ### Added +- `--csv-rows` filter option added to enable line-by-line transformation of CSV data +- `--json-output` option (Omeka specific) added to output JSON instead of posting to Omeka S +- `--proceed` option added to enable restarting of post from a checkpoint ### Changed +- Omeka S posting pipeline refactored to, among other things: + - introduce `OmekaContext` dataclass to encapsulate shared runtime state into a single object + - restructure overrides and update example files + - add filter from Ruby side (`--update`) + - remove all output JSON as part of a complete post to Omeka S + - adjust params for Omeka S API calls (see [`omeka_s_tools` fork](https://github.com/CDRH/omeka_s_tools)) + - update dependency list and create `requirements.in` file from which `requirements.txt` can be auto-generated + - improve logging and error handling + - fix bugs +- Updated Ruby and Python dependencies where possible, removed unused/deprecated dependencies +- Improved error handling: clarify XML/CSV parse errors and environment errors, show all errors at end of post +- Created option and fall-back to `full` for iiif URL creation ### Removed +- Removed `RestClient` in favor of built-in `net-http` library ### Migration - -### Deprecated +- Shifted to use Python `saxonche` package instead of Saxon calls to eliminate per-file JVM startup ### Security +- `safe_load` added for YAML +- SSL verification and regex validation added + +Note that several of the above changes and some of the language and description in the documentation were created with Claude Code. All code and documentation has been reviewed and updated by CDRH staff. ## [v1.1.0] - Omeka S Posting [v1.1.0]: https://github.com/CDRH/datura/compare/v1.0.1...v1.1.0 From 7af5e5450e5692ab09ba672a25362006bd8afbf8 Mon Sep 17 00:00:00 2001 From: nichgray Date: Mon, 17 Aug 2026 14:17:32 -0500 Subject: [PATCH 222/222] remove date placeholder from changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01ffab105..1437c0a4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ Versioning](https://semver.org/spec/v2.0.0.html). ### Security --> -## [v2.0.0] - 2026-08-20 - Shift to saxonche, improve error handling, refactor Omeka S posting, add post options +## [v2.0.0] - Shift to saxonche, improve error handling, refactor Omeka S posting, add post options [v2.0.0]: https://github.com/CDRH/datura/compare/v1.1.0...v2.0.0 ### Added