diff --git a/app/jobs/marc_export_job.rb b/app/jobs/marc_export_job.rb index 3d24cda7..86e9b942 100644 --- a/app/jobs/marc_export_job.rb +++ b/app/jobs/marc_export_job.rb @@ -4,11 +4,15 @@ class MarcExportJob < ActiveJob::Base def perform(theses) marc_filename = "#{filename}.mrc" zip_filename = "#{filename}.zip" + catalog_filename = "#{filename}.json" + begin - zip_file = MarcBatch.new(theses, marc_filename, zip_filename).build - BatchMailer.marc_batch_email(zip_filename, zip_file, theses).deliver_now + marc_zip_file = MarcBatch.new(theses, marc_filename, zip_filename).build + catalog_file = CatalogBatch.new(theses, catalog_filename).build + BatchMailer.marc_batch_email(zip_filename, marc_zip_file, catalog_filename, catalog_file, theses).deliver_now ensure - zip_file&.close + marc_zip_file&.close! + catalog_file&.close! end end diff --git a/app/mailers/batch_mailer.rb b/app/mailers/batch_mailer.rb index b4c7d206..3d6c5e97 100644 --- a/app/mailers/batch_mailer.rb +++ b/app/mailers/batch_mailer.rb @@ -1,13 +1,14 @@ class BatchMailer < ApplicationMailer - def marc_batch_email(marc_zip_filename, marc_zip_file, theses) + def marc_batch_email(marc_zip_filename, marc_zip_file, json_filename, json_file, theses) return unless ENV.fetch('DISABLE_ALL_EMAIL', 'true') == 'false' # allows PR builds to disable emails @theses = theses attachments[marc_zip_filename.to_s] = File.binread(marc_zip_file) + attachments[json_filename.to_s] = File.read(json_file) mail(from: "MIT Libraries <#{ENV['ETD_APP_EMAIL']}>", to: ENV['METADATA_ADMIN_EMAIL'], cc: ENV['MAINTAINER_EMAIL'], - subject: 'ETD MARC batch export') + subject: 'ETD metadata batch export') end def proquest_export_email(json_blob, csv_blob, thesis_count, budget_report_count) diff --git a/app/models/catalog_batch.rb b/app/models/catalog_batch.rb new file mode 100644 index 00000000..8d74cf2e --- /dev/null +++ b/app/models/catalog_batch.rb @@ -0,0 +1,37 @@ +# Generates a JSON metadata file from a collection of theses to add to the Libraries catalog. +# +# Produces a tempfile containing a JSON object with a 'theses' array, where each thesis is +# exported via CatalogExporter. +# +# Example: +# batch = CatalogBatch.new(theses_array, 'export.json') +# catalog_file = batch.build +# File.write('export.json', File.read(catalog_file.path)) +# catalog_file.close! # Clean up tempfile +class CatalogBatch + def initialize(theses, filename) + @theses = theses + @filename = filename + end + + # Builds and returns a Tempfile containing the JSON metadata export. The file is ready to read + # (file pointer rewound after writing). Caller is responsible for closing the file. + def build + catalog_file = Tempfile.new(@filename) + write_catalog_file(catalog_file) + catalog_file + end + + private + + def write_catalog_file(catalog_file) + theses_data = @theses.map do |thesis| + CatalogExporter.new(thesis).to_hash + end + + json_output = { theses: theses_data } + + catalog_file.write(JSON.pretty_generate(json_output)) + catalog_file.rewind + end +end diff --git a/app/models/catalog_exporter.rb b/app/models/catalog_exporter.rb new file mode 100644 index 00000000..d77fd4fa --- /dev/null +++ b/app/models/catalog_exporter.rb @@ -0,0 +1,72 @@ +# Exports a single thesis as a hash for JSON serialization. +# +# Transforms a thesis record into a flat-ish structure with nested arrays for repeating fields +# (authors, advisors, degrees, departments). +# +# Example: +# exporter = CatalogExporter.new(thesis) +# hash = exporter.to_hash +# # => { title: "...", abstract: "...", authors: [{name: "..."}, ...], ... } +class CatalogExporter + def initialize(thesis) + @thesis = thesis + end + + # Returns a hash representation of the thesis with all fields required by the metadata team. + # Includes: title, abstract, graduation_year, dspace_url, advisors, authors, degrees, and + # departments. Array fields are normalized to hashes with relevant metadata. + def to_hash + { + abstract:, + advisors:, + authors:, + degrees:, + departments:, + dspace_url:, + graduation_year:, + title: + } + end + + private + + def abstract + @thesis.abstract + end + + def advisors + @thesis.advisors.map do |advisor| + { name: advisor.name } + end + end + + def authors + @thesis.authors.map do |author| + { name: author.user.preferred_name } + end + end + + def degrees + @thesis.degrees.map do |degree| + { abbreviation: degree.abbreviation } + end + end + + def departments + @thesis.departments.map do |department| + { name: department.name_dspace } + end + end + + def dspace_url + "https://dspace.mit.edu/handle/#{@thesis.dspace_handle}" + end + + def graduation_year + @thesis.graduation_year + end + + def title + @thesis.title.squish + end +end diff --git a/app/views/batch_mailer/marc_batch_email.html.erb b/app/views/batch_mailer/marc_batch_email.html.erb index 0c5ae7b2..336a34bb 100644 --- a/app/views/batch_mailer/marc_batch_email.html.erb +++ b/app/views/batch_mailer/marc_batch_email.html.erb @@ -1,6 +1,8 @@
Hello,
Attached is a metadata export of <%= @theses.count %> theses generated on -<%= Date.current.strftime('%A, %B %d, %Y') %> at <%= Time.now.strftime('%r %Z') %>. +<%= Date.current.strftime('%A, %B %d, %Y') %> at <%= Time.now.strftime('%r %Z') %>.
+ +This export includes both MARC (in zip) and JSON.
Please contact the ETD team at <%= ENV['THESIS_ADMIN_EMAIL'] %> with any questions.
diff --git a/lib/tasks/metadata.rake b/lib/tasks/metadata.rake new file mode 100644 index 00000000..64f385cd --- /dev/null +++ b/lib/tasks/metadata.rake @@ -0,0 +1,49 @@ +namespace :metadata do + desc 'Generate a catalog export of a single published thesis for debugging' + task :catalog_export_thesis, [:thesis_id] => :environment do |_t, args| + if args.thesis_id.blank? + puts 'No thesis ID provided.' + next + end + + thesis = Thesis.find(args.thesis_id) + + if thesis.publication_status == 'Published' + catalog_exporter = CatalogExporter.new(thesis) + json_data = catalog_exporter.to_hash + + puts "Catalog Export for Thesis #{args.thesis_id}:" + puts JSON.pretty_generate(json_data) + else + puts "Thesis status of #{thesis.publication_status} cannot be exported. Only published theses can be exported." + end + end + + # This task is recommended for local development only. On Heroku (or other ephemeral filesystems), + # files saved to disk will be deleted when the dyno restarts, making them inaccessible. + desc 'Generate a catalog export batch for a specific term (e.g., "2024-June") and save to temp file' + task :catalog_export_batch, %i[term output_file] => :environment do |_t, args| + if args.term.blank? + puts 'Usage: rake metadata:catalog_export_batch["2024-June","output.json"]' + puts 'Term format: YYYY-Month (e.g., 2024-June, 2024-September)' + next + end + + year, month_name = args.term.split('-') + query_date = Date.parse("1 #{month_name} #{year}") + + output_file = args.output_file || Rails.root.join("tmp/catalog_export_#{args.term}_#{DateTime.now.utc.strftime('%H_%M')}.json").to_s + + theses = Thesis.published.where(grad_date: query_date.all_month) + + if theses.any? + catalog_batch = CatalogBatch.new(theses, File.basename(output_file)) + catalog_file = catalog_batch.build + FileUtils.cp(catalog_file.path, output_file) + catalog_file.close! + puts "Exported #{theses.count} theses to: #{output_file}" + else + puts "No published theses found for #{args.term}" + end + end +end diff --git a/test/jobs/marc_export_job_test.rb b/test/jobs/marc_export_job_test.rb index a9b68dba..b62ae06e 100644 --- a/test/jobs/marc_export_job_test.rb +++ b/test/jobs/marc_export_job_test.rb @@ -21,4 +21,29 @@ class MarcExportJobTest < ActiveJob::TestCase end end end + + test 'sent email includes both MARC and JSON attachments' do + ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do + theses = [theses(:one)] + Timecop.freeze(Time.utc(2022, 2, 14, 17, 10, 0)) do + email = MarcExportJob.perform_now(theses) + assert_equal 2, email.attachments.count + filenames = email.attachments.map(&:filename) + assert(filenames.include?('marc_220214_17_10.zip')) + assert(filenames.include?('marc_220214_17_10.json')) + end + end + end + + test 'JSON attachment is valid JSON' do + ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do + theses = [theses(:one)] + email = MarcExportJob.perform_now(theses) + json_attachment = email.attachments.find { |a| a.filename.ends_with?('.json') } + assert_not_nil(json_attachment) + + json_data = JSON.parse(json_attachment.body.to_s) + assert(json_data.key?('theses')) + end + end end diff --git a/test/mailers/batch_mailer_test.rb b/test/mailers/batch_mailer_test.rb index d99d8dc3..6d2d019f 100644 --- a/test/mailers/batch_mailer_test.rb +++ b/test/mailers/batch_mailer_test.rb @@ -4,19 +4,23 @@ class BatchMailerTest < ActionMailer::TestCase test 'sends emails for MARC batch exports' do ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do theses = [theses(:one), theses(:two)] - zip_file = MarcBatch.new(theses, 'marc.xml', 'marc.zip').build - email = BatchMailer.marc_batch_email('marc.zip', zip_file, theses) + marc_zip_file = MarcBatch.new(theses, 'marc.xml', 'marc.zip').build + catalog_file = CatalogBatch.new(theses, 'marc.json').build + email = BatchMailer.marc_batch_email('marc.zip', marc_zip_file, 'marc.json', catalog_file, theses) # Send the email, then test that it got queued assert_emails 1 do email.deliver_now end - # Make sure it was sent to the right person with the expected attachment. + # Make sure it was sent to the right person with the expected attachments. assert_equal ['app@example.com'], email.from assert_equal ['test-metadata@example.com'], email.to - assert_equal 'ETD MARC batch export', email.subject - assert_equal 'marc.zip', email.attachments.first.filename + assert_equal 'ETD metadata batch export', email.subject + assert_equal 2, email.attachments.count + filenames = email.attachments.map(&:filename) + assert_includes filenames, 'marc.zip' + assert_includes filenames, 'marc.json' assert_includes '2 theses', email.body.to_s end end @@ -24,8 +28,9 @@ class BatchMailerTest < ActionMailer::TestCase test 'zip file is attached with correct mimetype' do ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do theses = [theses(:one), theses(:two)] - zip_file = MarcBatch.new(theses, 'marc.xml', 'marc.zip').build - email = BatchMailer.marc_batch_email('marc.zip', zip_file, theses) + marc_zip_file = MarcBatch.new(theses, 'marc.xml', 'marc.zip').build + catalog_file = CatalogBatch.new(theses, 'marc.json').build + email = BatchMailer.marc_batch_email('marc.zip', marc_zip_file, 'marc.json', catalog_file, theses) attachment = email.attachments['marc.zip'] assert_equal 'application/zip; filename=marc.zip', attachment.content_type end diff --git a/test/models/catalog_batch_test.rb b/test/models/catalog_batch_test.rb new file mode 100644 index 00000000..4c6917cb --- /dev/null +++ b/test/models/catalog_batch_test.rb @@ -0,0 +1,73 @@ +require 'test_helper' + +class CatalogBatchTest < ActiveSupport::TestCase + test 'builds a valid JSON file' do + theses = [theses(:published)] + batch = CatalogBatch.new(theses, 'test.json') + catalog_file = batch.build + + json_content = File.read(catalog_file.path) + json_data = JSON.parse(json_content) + + assert_not_nil(json_data) + catalog_file.close + end + + test 'wraps theses in a wrapper object with theses key' do + theses = [theses(:published)] + batch = CatalogBatch.new(theses, 'test.json') + catalog_file = batch.build + + json_content = File.read(catalog_file.path) + json_data = JSON.parse(json_content) + + assert(json_data.key?('theses')) + assert(json_data['theses'].is_a?(Array)) + catalog_file.close + end + + test 'includes all theses in the batch' do + theses = [theses(:published), theses(:one)] + batch = CatalogBatch.new(theses, 'test.json') + catalog_file = batch.build + + json_content = File.read(catalog_file.path) + json_data = JSON.parse(json_content) + + assert_equal(2, json_data['theses'].count) + catalog_file.close + end + + test 'includes all required fields' do + theses = [theses(:published)] + batch = CatalogBatch.new(theses, 'test.json') + catalog_file = batch.build + + json_content = File.read(catalog_file.path) + json_data = JSON.parse(json_content) + + thesis_data = json_data['theses'].first + + assert(thesis_data.key?('abstract')) + assert(thesis_data.key?('advisors')) + assert(thesis_data.key?('authors')) + assert(thesis_data.key?('degrees')) + assert(thesis_data.key?('departments')) + assert(thesis_data.key?('dspace_url')) + assert(thesis_data.key?('graduation_year')) + assert(thesis_data.key?('title')) + + catalog_file.close + end + + test 'empty theses array produces valid JSON' do + batch = CatalogBatch.new([], 'test.json') + catalog_file = batch.build + + json_content = File.read(catalog_file.path) + json_data = JSON.parse(json_content) + + assert_equal(0, json_data['theses'].count) + catalog_file.close + end +end diff --git a/test/models/catalog_exporter_test.rb b/test/models/catalog_exporter_test.rb new file mode 100644 index 00000000..2903d910 --- /dev/null +++ b/test/models/catalog_exporter_test.rb @@ -0,0 +1,71 @@ +require 'test_helper' + +class CatalogExporterTest < ActiveSupport::TestCase + test 'includes correctly formatted title' do + thesis = theses(:published) + thesis.title = " A weirdly \n spaced title " + thesis.save + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert_equal "A weirdly spaced title", json_hash[:title] + end + + test 'includes abstract' do + thesis = theses(:published) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert_equal thesis.abstract, json_hash[:abstract] + end + + test 'includes grad year' do + thesis = theses(:published) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert_equal thesis.graduation_year, json_hash[:graduation_year] + end + + test 'includes correctly formatted dspace_url' do + thesis = theses(:published) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + expected_url = "https://dspace.mit.edu/handle/#{thesis.dspace_handle}" + assert_equal expected_url, json_hash[:dspace_url] + end + + test 'includes authors with nested structure' do + thesis = theses(:published) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert json_hash[:authors].is_a?(Array) + assert json_hash[:authors].first.is_a?(Hash) + assert json_hash[:authors].first[:name] + end + + test 'includes degrees with nested structure' do + thesis = theses(:published) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert json_hash[:degrees].is_a?(Array) + assert json_hash[:degrees].first.is_a?(Hash) + assert json_hash[:degrees].first[:abbreviation] + end + + test 'includes advisors with nested structure' do + thesis = theses(:published) + thesis.advisors << advisors(:first) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert json_hash[:advisors].is_a?(Array) + assert json_hash[:advisors].first.is_a?(Hash) + assert json_hash[:advisors].first[:name] + end + + test 'includes departments with nested structure' do + thesis = theses(:published) + exporter = CatalogExporter.new(thesis) + json_hash = exporter.to_hash + assert json_hash[:departments].is_a?(Array) + assert json_hash[:departments].first.is_a?(Hash) + assert json_hash[:departments].first[:name] + end +end