From 244beb25123c2bbb53900d4643446e85bd487c80 Mon Sep 17 00:00:00 2001
From: kennethrioja <59597207+kennethrioja@users.noreply.github.com>
Date: Tue, 29 Sep 2026 16:44:08 +0200
Subject: [PATCH] feat(epic-tutorials-ingestor): new heptraining ingestor and
tests
---
app/models/source_filter.rb | 184 +++++++++---------
lib/ingestors/cds_videos_ingestor.rb | 142 --------------
lib/ingestors/github_ingestor.rb | 5 +-
.../heptraining/cds_videos_ingestor.rb | 144 ++++++++++++++
.../heptraining/epic_tutorials_ingestor.rb | 89 +++++++++
lib/ingestors/ingestor_factory.rb | 7 +-
.../heptraining/epictutorials/epic-page.md | 35 ++++
.../cds_videos_ingestor_test.rb | 4 +-
.../epic_tutorials_ingestor_test.rb | 134 +++++++++++++
9 files changed, 503 insertions(+), 241 deletions(-)
delete mode 100644 lib/ingestors/cds_videos_ingestor.rb
create mode 100644 lib/ingestors/heptraining/cds_videos_ingestor.rb
create mode 100644 lib/ingestors/heptraining/epic_tutorials_ingestor.rb
create mode 100644 test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md
rename test/unit/ingestors/{ => heptraining}/cds_videos_ingestor_test.rb (97%)
create mode 100644 test/unit/ingestors/heptraining/epic_tutorials_ingestor_test.rb
diff --git a/app/models/source_filter.rb b/app/models/source_filter.rb
index a36503494..93df3bb0a 100644
--- a/app/models/source_filter.rb
+++ b/app/models/source_filter.rb
@@ -18,100 +18,98 @@ def self.array_string_match(content_value, filter_value)
end
end
-
-FILTER_DEFINITIONS = {
- 'target_audience' => {
- comparison: FilterComparisons.method(:array_string_match)
- },
-
- 'keyword' => {
- comparison: FilterComparisons.method(:array_string_match),
- filter_property: 'keywords'
- },
-
- 'title' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'title_contains' => {
- comparison: FilterComparisons.method(:contains_string_match),
- filter_property: 'title'
- },
-
- 'description' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'description_contains' => {
- comparison: FilterComparisons.method(:contains_string_match),
- filter_property: 'description'
- },
-
- 'url' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'url_prefix' => {
- comparison: FilterComparisons.method(:prefix_string_match),
- filter_property: 'url'
- },
-
- 'doi' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'license' => {
- comparison: FilterComparisons.method(:string_match),
- filter_property: 'licence'
- },
-
- 'difficulty_level' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'resource_type' => {
- comparison: FilterComparisons.method(:array_string_match)
- },
-
- 'prerequisites_contains' => {
- comparison: FilterComparisons.method(:contains_string_match),
- filter_property: 'prerequisites'
- },
-
- 'learning_objectives_contains' => {
- comparison: FilterComparisons.method(:contains_string_match),
- filter_property: 'learning_objectives'
- },
-
- 'subtitle' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'subtitle_contains' => {
- comparison: FilterComparisons.method(:contains_string_match),
- filter_property: 'subtitle'
- },
-
- 'city' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'country' => {
- comparison: FilterComparisons.method(:string_match)
- },
-
- 'event_type' => {
- comparison: FilterComparisons.method(:array_string_match),
- filter_property: 'event_types'
- },
-
- 'timezone' => {
- comparison: FilterComparisons.method(:string_match)
- }
-}.freeze
-
-
class SourceFilter < ApplicationRecord
+ FILTER_DEFINITIONS = {
+ 'target_audience' => {
+ comparison: FilterComparisons.method(:array_string_match)
+ },
+
+ 'keyword' => {
+ comparison: FilterComparisons.method(:array_string_match),
+ filter_property: 'keywords'
+ },
+
+ 'title' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'title_contains' => {
+ comparison: FilterComparisons.method(:contains_string_match),
+ filter_property: 'title'
+ },
+
+ 'description' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'description_contains' => {
+ comparison: FilterComparisons.method(:contains_string_match),
+ filter_property: 'description'
+ },
+
+ 'url' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'url_prefix' => {
+ comparison: FilterComparisons.method(:prefix_string_match),
+ filter_property: 'url'
+ },
+
+ 'doi' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'license' => {
+ comparison: FilterComparisons.method(:string_match),
+ filter_property: 'licence'
+ },
+
+ 'difficulty_level' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'resource_type' => {
+ comparison: FilterComparisons.method(:array_string_match)
+ },
+
+ 'prerequisites_contains' => {
+ comparison: FilterComparisons.method(:contains_string_match),
+ filter_property: 'prerequisites'
+ },
+
+ 'learning_objectives_contains' => {
+ comparison: FilterComparisons.method(:contains_string_match),
+ filter_property: 'learning_objectives'
+ },
+
+ 'subtitle' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'subtitle_contains' => {
+ comparison: FilterComparisons.method(:contains_string_match),
+ filter_property: 'subtitle'
+ },
+
+ 'city' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'country' => {
+ comparison: FilterComparisons.method(:string_match)
+ },
+
+ 'event_type' => {
+ comparison: FilterComparisons.method(:array_string_match),
+ filter_property: 'event_types'
+ },
+
+ 'timezone' => {
+ comparison: FilterComparisons.method(:string_match)
+ }
+ }.freeze
+
belongs_to :source
auto_strip_attributes :value
diff --git a/lib/ingestors/cds_videos_ingestor.rb b/lib/ingestors/cds_videos_ingestor.rb
deleted file mode 100644
index 6376a1d34..000000000
--- a/lib/ingestors/cds_videos_ingestor.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-# frozen_string_literal: true
-require 'json'
-
-module Ingestors
- class CdsVideosIngestor < Ingestor # rubocop:disable Style/Documentation
- CDS_VIDEOS_RECORD_URL = 'https://videos.cern.ch/record/'
-
- def self.config
- {
- key: 'cds videos',
- title: 'CDS videos record or search',
- category: :materials,
- user_agent: 'TeSS CDS Videos ingestor'
- }
- end
-
- # Reads from single CDS record URL or search query URL
- # For a search query, we need to change fields: page to 1 and size to 100
- # Then read all pages while running `to_material()`
- # Else for a single material, go direct with `to_material()`
- def read(source_url)
- api_url = to_api(source_url)
-
- if source_url.include?('search')
- api_url_page_updated = update_url_field_value(api_url, 'page', 1)
- api_url_page_size_updated = update_url_field_value(api_url_page_updated, 'size', 100)
- data = JSON.parse(open_url(api_url_page_size_updated).read)
- add_records_material(data)
- else
- data = JSON.parse(open_url(api_url).read)
- add_material to_material(data)
- end
- rescue StandardError => e
- Rails.logger.error("#{e.class}: read() failed, #{e.message}")
- end
-
- private
-
- def to_api(url)
- uri = URI(url)
- parts = uri.path.split('/') # 'example.com/foo/bar' will have path == '/foo/bar', so three parts
-
- # FROM '/search?{query}'
- if parts[1] == 'search' && parts.size == 2
- # TO '/api/records/{query}'
- "https://#{uri.host}/api/records?#{uri.query}"
- # FROM '/record/{recid}'
- elsif parts[1] == 'record' && parts.size == 3
- # TO 'api/record/{recid}'
- "https://#{uri.host}/api/#{parts[1]}/#{parts[2]}"
- end
- end
-
- def update_url_field_value(url, field, value)
- uri = URI.parse(url)
- params = URI.decode_www_form(uri.query || '').to_h
- params[field] = value.to_s
- uri.query = URI.encode_www_form(params)
- uri.to_s
- end
-
- def add_records_material(initial_data)
- data = initial_data
-
- while data
- hits = data.dig('hits', 'hits') || []
- hits.each { |hit| add_material to_material(hit) }
-
- next_url = data.dig('links', 'next')
- break unless next_url
-
- response = open_url(next_url)&.read
- data = response ? JSON.parse(response) : nil
- end
- end
-
- # Sets material hash keys and values and add them to material
- def to_material(data)
- metadata = (data['metadata'] || {}).transform_keys { |k| k == '_cds' ? 'cds' : k }
-
- material = OpenStruct.new
- material.title = metadata.dig('title', 'title').to_s.titleize
- material.url = "#{CDS_VIDEOS_RECORD_URL}#{data['id']}"
- material.description = get_description(metadata)
- material.keywords = Array(metadata['keywords']).first(10).map { |k| k['name'] }.compact
- material.licence = metadata.dig('copyright', 'holder') == 'CERN' ? 'other-at' : 'notspecified'
- material.status = 'Active'
- material.contact = get_contact(metadata)
- material.version = Array(metadata['report_number']).first
- material.date_created = metadata['date']
- material.date_published = metadata['publication_date']
- material.authors = get_authors(metadata)
- material.contributors = get_contributors(metadata)
- type = metadata['type']
- category = metadata['category']
- resource_type = [type, category].compact.map { |v| v.to_s.titleize }.join(' – ')
- material.resource_type = resource_type.present? ? [resource_type] : []
- material
- end
-
- def get_description(metadata)
- description = metadata['description'].split('
')[0] + "\n\n"
- description = description&.<< "Video duration: #{metadata['duration']}\n\n" if metadata['duration']
- description = description + metadata['additional_descriptions']&.map { |k| k['description'] }&.join("\n\n") + "\n\n" || '' if metadata['additional_descriptions']
- description = description&.<< "Related identifier:\n- Is part of: #{metadata['alternate_identifiers'][0]['value']} (URL)\n\n" if metadata['alternate_identifiers']&.any?
- description = description&.<< "Copyright: [#{metadata.dig('copyright', 'holder')}](#{metadata.dig('copyright', 'url')}) (#{metadata.dig('copyright', 'year')})" if metadata.dig('copyright', 'holder')
- description
- end
-
- def get_authors(metadata)
- metadata['contributors']
- &.select { |c| c['role'] == 'Creator' }
- &.reject { |c| c['name'].downcase == 'cern' }
- &.map { |c| c['name']&.gsub(',', '')&.titleize } || ''
- end
-
- def get_contributors(metadata)
- metadata['contributors']
- &.reject { |c| c['role'] == 'Creator' || c['role'] == 'ContactPerson' }
- &.reject { |c| c['name'].downcase == 'cern' }
- &.map { |c| c['name']&.gsub(',', '')&.titleize } || ''
- end
-
- def get_contact(metadata)
- contributors = metadata['contributors'] || []
-
- # 1. Check for ContactPerson name
- contact_person = contributors.find { |c| c['role'] == 'ContactPerson' }
- contact_name = contact_person&.[]('name')&.gsub(',', '')&.titleize
-
- # 2. Check for Creator email
- creator = contributors.find { |c| c['role'] == 'Creator' }
- creator_email = creator&.[]('email')&.gsub(',', '')&.titleize
-
- # 3. Check for any other contributor email
- other_contributor = contributors.find { |c| !%w[ContactPerson Creator].include?(c['role']) }
- other_email = other_contributor&.[]('email')
-
- contact_name.presence || creator_email.presence || other_email || ''
- end
- end
-end
diff --git a/lib/ingestors/github_ingestor.rb b/lib/ingestors/github_ingestor.rb
index 6cc65eafc..77f29bc75 100644
--- a/lib/ingestors/github_ingestor.rb
+++ b/lib/ingestors/github_ingestor.rb
@@ -93,7 +93,10 @@ def github_api_from_io(uri, parts)
# ttl: time-to-live in seconds (default 7 days)
def cache_fetch(key, url)
Rails.cache.fetch(key, expires_in: TTL, skip_nil: true) do
- JSON.parse(open_url(url).read)
+ response = open_url(url)
+ next nil unless response
+
+ JSON.parse(response.read)
end
end
diff --git a/lib/ingestors/heptraining/cds_videos_ingestor.rb b/lib/ingestors/heptraining/cds_videos_ingestor.rb
new file mode 100644
index 000000000..a4f145c79
--- /dev/null
+++ b/lib/ingestors/heptraining/cds_videos_ingestor.rb
@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+require 'json'
+
+module Ingestors
+ module Heptraining
+ class CdsVideosIngestor < Ingestor # rubocop:disable Style/Documentation
+ CDS_VIDEOS_RECORD_URL = 'https://videos.cern.ch/record/'
+
+ def self.config
+ {
+ key: 'cds videos',
+ title: 'CDS videos record or search',
+ category: :materials,
+ user_agent: 'TeSS CDS Videos ingestor'
+ }
+ end
+
+ # Reads from single CDS record URL or search query URL
+ # For a search query, we need to change fields: page to 1 and size to 100
+ # Then read all pages while running `to_material()`
+ # Else for a single material, go direct with `to_material()`
+ def read(source_url)
+ api_url = to_api(source_url)
+
+ if source_url.include?('search')
+ api_url_page_updated = update_url_field_value(api_url, 'page', 1)
+ api_url_page_size_updated = update_url_field_value(api_url_page_updated, 'size', 100)
+ data = JSON.parse(open_url(api_url_page_size_updated).read)
+ add_records_material(data)
+ else
+ data = JSON.parse(open_url(api_url).read)
+ add_material to_material(data)
+ end
+ rescue StandardError => e
+ Rails.logger.error("#{e.class}: read() failed, #{e.message}")
+ end
+
+ private
+
+ def to_api(url)
+ uri = URI(url)
+ parts = uri.path.split('/') # 'example.com/foo/bar' will have path == '/foo/bar', so three parts
+
+ # FROM '/search?{query}'
+ if parts[1] == 'search' && parts.size == 2
+ # TO '/api/records/{query}'
+ "https://#{uri.host}/api/records?#{uri.query}"
+ # FROM '/record/{recid}'
+ elsif parts[1] == 'record' && parts.size == 3
+ # TO 'api/record/{recid}'
+ "https://#{uri.host}/api/#{parts[1]}/#{parts[2]}"
+ end
+ end
+
+ def update_url_field_value(url, field, value)
+ uri = URI.parse(url)
+ params = URI.decode_www_form(uri.query || '').to_h
+ params[field] = value.to_s
+ uri.query = URI.encode_www_form(params)
+ uri.to_s
+ end
+
+ def add_records_material(initial_data)
+ data = initial_data
+
+ while data
+ hits = data.dig('hits', 'hits') || []
+ hits.each { |hit| add_material to_material(hit) }
+
+ next_url = data.dig('links', 'next')
+ break unless next_url
+
+ response = open_url(next_url)&.read
+ data = response ? JSON.parse(response) : nil
+ end
+ end
+
+ # Sets material hash keys and values and add them to material
+ def to_material(data)
+ metadata = (data['metadata'] || {}).transform_keys { |k| k == '_cds' ? 'cds' : k }
+
+ material = OpenStruct.new
+ material.title = metadata.dig('title', 'title').to_s.titleize
+ material.url = "#{CDS_VIDEOS_RECORD_URL}#{data['id']}"
+ material.description = get_description(metadata)
+ material.keywords = Array(metadata['keywords']).first(10).map { |k| k['name'] }.compact
+ material.licence = metadata.dig('copyright', 'holder') == 'CERN' ? 'other-at' : 'notspecified'
+ material.status = 'Active'
+ material.contact = get_contact(metadata)
+ material.version = Array(metadata['report_number']).first
+ material.date_created = metadata['date']
+ material.date_published = metadata['publication_date']
+ material.authors = get_authors(metadata)
+ material.contributors = get_contributors(metadata)
+ type = metadata['type']
+ category = metadata['category']
+ resource_type = [type, category].compact.map { |v| v.to_s.titleize }.join(' – ')
+ material.resource_type = resource_type.present? ? [resource_type] : []
+ material
+ end
+
+ def get_description(metadata)
+ description = metadata['description'].split('
')[0] + "\n\n"
+ description = description&.<< "Video duration: #{metadata['duration']}\n\n" if metadata['duration']
+ description = description + metadata['additional_descriptions']&.map { |k| k['description'] }&.join("\n\n") + "\n\n" || '' if metadata['additional_descriptions']
+ description = description&.<< "Related identifier:\n- Is part of: #{metadata['alternate_identifiers'][0]['value']} (URL)\n\n" if metadata['alternate_identifiers']&.any?
+ description = description&.<< "Copyright: [#{metadata.dig('copyright', 'holder')}](#{metadata.dig('copyright', 'url')}) (#{metadata.dig('copyright', 'year')})" if metadata.dig('copyright', 'holder')
+ description
+ end
+
+ def get_authors(metadata)
+ metadata['contributors']
+ &.select { |c| c['role'] == 'Creator' }
+ &.reject { |c| c['name'].downcase == 'cern' }
+ &.map { |c| c['name']&.gsub(',', '')&.titleize } || ''
+ end
+
+ def get_contributors(metadata)
+ metadata['contributors']
+ &.reject { |c| c['role'] == 'Creator' || c['role'] == 'ContactPerson' }
+ &.reject { |c| c['name'].downcase == 'cern' }
+ &.map { |c| c['name']&.gsub(',', '')&.titleize } || ''
+ end
+
+ def get_contact(metadata)
+ contributors = metadata['contributors'] || []
+
+ # 1. Check for ContactPerson name
+ contact_person = contributors.find { |c| c['role'] == 'ContactPerson' }
+ contact_name = contact_person&.[]('name')&.gsub(',', '')&.titleize
+
+ # 2. Check for Creator email
+ creator = contributors.find { |c| c['role'] == 'Creator' }
+ creator_email = creator&.[]('email')&.gsub(',', '')&.titleize
+
+ # 3. Check for any other contributor email
+ other_contributor = contributors.find { |c| !%w[ContactPerson Creator].include?(c['role']) }
+ other_email = other_contributor&.[]('email')
+
+ contact_name.presence || creator_email.presence || other_email || ''
+ end
+ end
+ end
+end
diff --git a/lib/ingestors/heptraining/epic_tutorials_ingestor.rb b/lib/ingestors/heptraining/epic_tutorials_ingestor.rb
new file mode 100644
index 000000000..1476ef3db
--- /dev/null
+++ b/lib/ingestors/heptraining/epic_tutorials_ingestor.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Ingestors
+ module Heptraining
+ class EpicTutorialsIngestor < Ingestor
+ def self.config
+ {
+ key: 'epic_tutorials',
+ title: 'ePIC Tutorials',
+ category: :materials,
+ user_agent: 'TeSS ePIC Tutorials ingestor'
+ }
+ end
+
+ def read(url)
+ @verbose = false
+ response = open_url(url, raise: true)
+ return unless response
+
+ process_frontmatter(response.read)
+ rescue StandardError => e
+ Rails.logger.error("#{e.class}: read() failed, #{e.message}")
+ end
+
+ private
+
+ def process_frontmatter(content)
+ yaml_match = content.match(/\A---\s*\n(.*?)\n?---\s*/m)
+ raise 'No YAML frontmatter found' unless yaml_match
+
+ data = YAML.safe_load(yaml_match[1], permitted_classes: [Date, Time])
+ tutorials = data['tutorials'] || []
+
+ tutorials.each do |tutorial|
+ process_tutorial(tutorial)
+ end
+ end
+
+ def process_tutorial(tutorial)
+ url = tutorial['url']
+ return if url.blank?
+
+ materials, events = fetch_from_other_ingestor(url)
+
+ # Build external resources list from YAML
+ ext_resources = (tutorial['resources'] || []).map do |res|
+ next if res['url'].blank?
+
+ kind = res['kind'].present? ? res['kind'].capitalize : nil
+ title = res['label'].present? ? "#{kind} #{res['label']}" : kind
+ { title: title, url: res['url'] }
+ end.compact
+
+ # Attach external resources to returned materials
+ (materials + events).each do |material|
+ material.external_resources ||= []
+ material.external_resources.concat(ext_resources)
+ end
+
+ @materials.concat(materials)
+ @events.concat(events)
+ end
+
+ def fetch_from_other_ingestor(url)
+ ingestor = if github_url?(url)
+ Ingestors::GithubIngestor.new
+ elsif indico_url?(url)
+ Ingestors::IndicoIngestor.new
+ end
+
+ return [[], []] unless ingestor
+
+ ingestor.read(url)
+ [ingestor.materials, ingestor.events]
+ end
+
+ def github_url?(url)
+ uri = URI.parse(url)
+ uri.host&.downcase == 'github.com' || uri.host&.downcase&.end_with?('.github.io')
+ end
+
+ def indico_url?(url)
+ url.include?('indico')
+ end
+ end
+ end
+end
\ No newline at end of file
diff --git a/lib/ingestors/ingestor_factory.rb b/lib/ingestors/ingestor_factory.rb
index d4c2d3fac..97e8c9cdb 100644
--- a/lib/ingestors/ingestor_factory.rb
+++ b/lib/ingestors/ingestor_factory.rb
@@ -15,8 +15,7 @@ def self.ingestors
Ingestors::GithubIngestor,
Ingestors::EventRSSIngestor,
Ingestors::MaterialRSSIngestor,
- Ingestors::YoutubeIngestor,
- Ingestors::CdsVideosIngestor
+ Ingestors::YoutubeIngestor
] + taxila_ingestors + llm_ingestors + heptraining_ingestors
end
@@ -57,7 +56,9 @@ def self.llm_ingestors
def self.heptraining_ingestors
[
- Ingestors::Heptraining::GrayScottIngestor
+ Ingestors::Heptraining::GrayScottIngestor,
+ Ingestors::Heptraining::CdsVideosIngestor,
+ Ingestors::Heptraining::EpicTutorialsIngestor
]
end
diff --git a/test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md b/test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md
new file mode 100644
index 000000000..56aabb26f
--- /dev/null
+++ b/test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md
@@ -0,0 +1,35 @@
+---
+title: ePIC Tutorials
+description: Tutorials for ePIC environment setup, analysis, simulation, reconstruction, file access, and benchmarks.
+name: tutorials
+layout: default
+tutorials:
+
+ - title: "Setting up an environment"
+ url: "https://github.com/hsf-training/cpluspluscourse"
+ summary: "How to get eic-shell and what it is
Usage of github"
+ image: "/assets/images/tutorials/tutorial-eicshell.png"
+ resources:
+ - label: "1"
+ url: "https://www.youtube.com/watch?v=Y0Mg24XLomY"
+ kind: "youtube"
+ - label: "2"
+ url: "https://www.youtube.com/watch?v=5HmzFnYW4W4"
+ kind: "youtube"
+
+ - title: "Reconstruction framework"
+ url: "https://github.com/hsf-training/cpluspluscourse"
+ summary: "eicrecon in a nutshell.
JANA2 framework , Plugins, Factories, and Algorithms"
+ image: "/assets/images/tutorials/tutorial-jana2.png"
+
+ - title: "Getting started with physics analysis"
+ url: "https://indico.bnl.gov/event/27123"
+ summary: "Understanding EPIC detector regions
Quality assurance plots for physics
Acceptance and efficiency
Beam effects and afterburner
Full-chain analysis"
+ image: "/assets/images/tutorials/tutorial-physics-analysis.png"
+ resources:
+ - url: "https://youtu.be/goONxXudL-s"
+ kind: "youtube"
+---
+
+