From 244beb25123c2bbb53900d4643446e85bd487c80 Mon Sep 17 00:00:00 2001 From: kennethrioja <59597207+kennethrioja@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:44:08 +0200 Subject: [PATCH] feat(epic-tutorials-ingestor): new heptraining ingestor and tests --- app/models/source_filter.rb | 184 +++++++++--------- lib/ingestors/cds_videos_ingestor.rb | 142 -------------- lib/ingestors/github_ingestor.rb | 5 +- .../heptraining/cds_videos_ingestor.rb | 144 ++++++++++++++ .../heptraining/epic_tutorials_ingestor.rb | 89 +++++++++ lib/ingestors/ingestor_factory.rb | 7 +- .../heptraining/epictutorials/epic-page.md | 35 ++++ .../cds_videos_ingestor_test.rb | 4 +- .../epic_tutorials_ingestor_test.rb | 134 +++++++++++++ 9 files changed, 503 insertions(+), 241 deletions(-) delete mode 100644 lib/ingestors/cds_videos_ingestor.rb create mode 100644 lib/ingestors/heptraining/cds_videos_ingestor.rb create mode 100644 lib/ingestors/heptraining/epic_tutorials_ingestor.rb create mode 100644 test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md rename test/unit/ingestors/{ => heptraining}/cds_videos_ingestor_test.rb (97%) create mode 100644 test/unit/ingestors/heptraining/epic_tutorials_ingestor_test.rb diff --git a/app/models/source_filter.rb b/app/models/source_filter.rb index a36503494..93df3bb0a 100644 --- a/app/models/source_filter.rb +++ b/app/models/source_filter.rb @@ -18,100 +18,98 @@ def self.array_string_match(content_value, filter_value) end end - -FILTER_DEFINITIONS = { - 'target_audience' => { - comparison: FilterComparisons.method(:array_string_match) - }, - - 'keyword' => { - comparison: FilterComparisons.method(:array_string_match), - filter_property: 'keywords' - }, - - 'title' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'title_contains' => { - comparison: FilterComparisons.method(:contains_string_match), - filter_property: 'title' - }, - - 'description' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'description_contains' => { - comparison: FilterComparisons.method(:contains_string_match), - filter_property: 'description' - }, - - 'url' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'url_prefix' => { - comparison: FilterComparisons.method(:prefix_string_match), - filter_property: 'url' - }, - - 'doi' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'license' => { - comparison: FilterComparisons.method(:string_match), - filter_property: 'licence' - }, - - 'difficulty_level' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'resource_type' => { - comparison: FilterComparisons.method(:array_string_match) - }, - - 'prerequisites_contains' => { - comparison: FilterComparisons.method(:contains_string_match), - filter_property: 'prerequisites' - }, - - 'learning_objectives_contains' => { - comparison: FilterComparisons.method(:contains_string_match), - filter_property: 'learning_objectives' - }, - - 'subtitle' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'subtitle_contains' => { - comparison: FilterComparisons.method(:contains_string_match), - filter_property: 'subtitle' - }, - - 'city' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'country' => { - comparison: FilterComparisons.method(:string_match) - }, - - 'event_type' => { - comparison: FilterComparisons.method(:array_string_match), - filter_property: 'event_types' - }, - - 'timezone' => { - comparison: FilterComparisons.method(:string_match) - } -}.freeze - - class SourceFilter < ApplicationRecord + FILTER_DEFINITIONS = { + 'target_audience' => { + comparison: FilterComparisons.method(:array_string_match) + }, + + 'keyword' => { + comparison: FilterComparisons.method(:array_string_match), + filter_property: 'keywords' + }, + + 'title' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'title_contains' => { + comparison: FilterComparisons.method(:contains_string_match), + filter_property: 'title' + }, + + 'description' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'description_contains' => { + comparison: FilterComparisons.method(:contains_string_match), + filter_property: 'description' + }, + + 'url' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'url_prefix' => { + comparison: FilterComparisons.method(:prefix_string_match), + filter_property: 'url' + }, + + 'doi' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'license' => { + comparison: FilterComparisons.method(:string_match), + filter_property: 'licence' + }, + + 'difficulty_level' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'resource_type' => { + comparison: FilterComparisons.method(:array_string_match) + }, + + 'prerequisites_contains' => { + comparison: FilterComparisons.method(:contains_string_match), + filter_property: 'prerequisites' + }, + + 'learning_objectives_contains' => { + comparison: FilterComparisons.method(:contains_string_match), + filter_property: 'learning_objectives' + }, + + 'subtitle' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'subtitle_contains' => { + comparison: FilterComparisons.method(:contains_string_match), + filter_property: 'subtitle' + }, + + 'city' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'country' => { + comparison: FilterComparisons.method(:string_match) + }, + + 'event_type' => { + comparison: FilterComparisons.method(:array_string_match), + filter_property: 'event_types' + }, + + 'timezone' => { + comparison: FilterComparisons.method(:string_match) + } + }.freeze + belongs_to :source auto_strip_attributes :value diff --git a/lib/ingestors/cds_videos_ingestor.rb b/lib/ingestors/cds_videos_ingestor.rb deleted file mode 100644 index 6376a1d34..000000000 --- a/lib/ingestors/cds_videos_ingestor.rb +++ /dev/null @@ -1,142 +0,0 @@ -# frozen_string_literal: true -require 'json' - -module Ingestors - class CdsVideosIngestor < Ingestor # rubocop:disable Style/Documentation - CDS_VIDEOS_RECORD_URL = 'https://videos.cern.ch/record/' - - def self.config - { - key: 'cds videos', - title: 'CDS videos record or search', - category: :materials, - user_agent: 'TeSS CDS Videos ingestor' - } - end - - # Reads from single CDS record URL or search query URL - # For a search query, we need to change fields: page to 1 and size to 100 - # Then read all pages while running `to_material()` - # Else for a single material, go direct with `to_material()` - def read(source_url) - api_url = to_api(source_url) - - if source_url.include?('search') - api_url_page_updated = update_url_field_value(api_url, 'page', 1) - api_url_page_size_updated = update_url_field_value(api_url_page_updated, 'size', 100) - data = JSON.parse(open_url(api_url_page_size_updated).read) - add_records_material(data) - else - data = JSON.parse(open_url(api_url).read) - add_material to_material(data) - end - rescue StandardError => e - Rails.logger.error("#{e.class}: read() failed, #{e.message}") - end - - private - - def to_api(url) - uri = URI(url) - parts = uri.path.split('/') # 'example.com/foo/bar' will have path == '/foo/bar', so three parts - - # FROM '/search?{query}' - if parts[1] == 'search' && parts.size == 2 - # TO '/api/records/{query}' - "https://#{uri.host}/api/records?#{uri.query}" - # FROM '/record/{recid}' - elsif parts[1] == 'record' && parts.size == 3 - # TO 'api/record/{recid}' - "https://#{uri.host}/api/#{parts[1]}/#{parts[2]}" - end - end - - def update_url_field_value(url, field, value) - uri = URI.parse(url) - params = URI.decode_www_form(uri.query || '').to_h - params[field] = value.to_s - uri.query = URI.encode_www_form(params) - uri.to_s - end - - def add_records_material(initial_data) - data = initial_data - - while data - hits = data.dig('hits', 'hits') || [] - hits.each { |hit| add_material to_material(hit) } - - next_url = data.dig('links', 'next') - break unless next_url - - response = open_url(next_url)&.read - data = response ? JSON.parse(response) : nil - end - end - - # Sets material hash keys and values and add them to material - def to_material(data) - metadata = (data['metadata'] || {}).transform_keys { |k| k == '_cds' ? 'cds' : k } - - material = OpenStruct.new - material.title = metadata.dig('title', 'title').to_s.titleize - material.url = "#{CDS_VIDEOS_RECORD_URL}#{data['id']}" - material.description = get_description(metadata) - material.keywords = Array(metadata['keywords']).first(10).map { |k| k['name'] }.compact - material.licence = metadata.dig('copyright', 'holder') == 'CERN' ? 'other-at' : 'notspecified' - material.status = 'Active' - material.contact = get_contact(metadata) - material.version = Array(metadata['report_number']).first - material.date_created = metadata['date'] - material.date_published = metadata['publication_date'] - material.authors = get_authors(metadata) - material.contributors = get_contributors(metadata) - type = metadata['type'] - category = metadata['category'] - resource_type = [type, category].compact.map { |v| v.to_s.titleize }.join(' – ') - material.resource_type = resource_type.present? ? [resource_type] : [] - material - end - - def get_description(metadata) - description = metadata['description'].split('
')[0] + "\n\n" - description = description&.<< "Video duration: #{metadata['duration']}\n\n" if metadata['duration'] - description = description + metadata['additional_descriptions']&.map { |k| k['description'] }&.join("\n\n") + "\n\n" || '' if metadata['additional_descriptions'] - description = description&.<< "Related identifier:\n- Is part of: #{metadata['alternate_identifiers'][0]['value']} (URL)\n\n" if metadata['alternate_identifiers']&.any? - description = description&.<< "Copyright: [#{metadata.dig('copyright', 'holder')}](#{metadata.dig('copyright', 'url')}) (#{metadata.dig('copyright', 'year')})" if metadata.dig('copyright', 'holder') - description - end - - def get_authors(metadata) - metadata['contributors'] - &.select { |c| c['role'] == 'Creator' } - &.reject { |c| c['name'].downcase == 'cern' } - &.map { |c| c['name']&.gsub(',', '')&.titleize } || '' - end - - def get_contributors(metadata) - metadata['contributors'] - &.reject { |c| c['role'] == 'Creator' || c['role'] == 'ContactPerson' } - &.reject { |c| c['name'].downcase == 'cern' } - &.map { |c| c['name']&.gsub(',', '')&.titleize } || '' - end - - def get_contact(metadata) - contributors = metadata['contributors'] || [] - - # 1. Check for ContactPerson name - contact_person = contributors.find { |c| c['role'] == 'ContactPerson' } - contact_name = contact_person&.[]('name')&.gsub(',', '')&.titleize - - # 2. Check for Creator email - creator = contributors.find { |c| c['role'] == 'Creator' } - creator_email = creator&.[]('email')&.gsub(',', '')&.titleize - - # 3. Check for any other contributor email - other_contributor = contributors.find { |c| !%w[ContactPerson Creator].include?(c['role']) } - other_email = other_contributor&.[]('email') - - contact_name.presence || creator_email.presence || other_email || '' - end - end -end diff --git a/lib/ingestors/github_ingestor.rb b/lib/ingestors/github_ingestor.rb index 6cc65eafc..77f29bc75 100644 --- a/lib/ingestors/github_ingestor.rb +++ b/lib/ingestors/github_ingestor.rb @@ -93,7 +93,10 @@ def github_api_from_io(uri, parts) # ttl: time-to-live in seconds (default 7 days) def cache_fetch(key, url) Rails.cache.fetch(key, expires_in: TTL, skip_nil: true) do - JSON.parse(open_url(url).read) + response = open_url(url) + next nil unless response + + JSON.parse(response.read) end end diff --git a/lib/ingestors/heptraining/cds_videos_ingestor.rb b/lib/ingestors/heptraining/cds_videos_ingestor.rb new file mode 100644 index 000000000..a4f145c79 --- /dev/null +++ b/lib/ingestors/heptraining/cds_videos_ingestor.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true +require 'json' + +module Ingestors + module Heptraining + class CdsVideosIngestor < Ingestor # rubocop:disable Style/Documentation + CDS_VIDEOS_RECORD_URL = 'https://videos.cern.ch/record/' + + def self.config + { + key: 'cds videos', + title: 'CDS videos record or search', + category: :materials, + user_agent: 'TeSS CDS Videos ingestor' + } + end + + # Reads from single CDS record URL or search query URL + # For a search query, we need to change fields: page to 1 and size to 100 + # Then read all pages while running `to_material()` + # Else for a single material, go direct with `to_material()` + def read(source_url) + api_url = to_api(source_url) + + if source_url.include?('search') + api_url_page_updated = update_url_field_value(api_url, 'page', 1) + api_url_page_size_updated = update_url_field_value(api_url_page_updated, 'size', 100) + data = JSON.parse(open_url(api_url_page_size_updated).read) + add_records_material(data) + else + data = JSON.parse(open_url(api_url).read) + add_material to_material(data) + end + rescue StandardError => e + Rails.logger.error("#{e.class}: read() failed, #{e.message}") + end + + private + + def to_api(url) + uri = URI(url) + parts = uri.path.split('/') # 'example.com/foo/bar' will have path == '/foo/bar', so three parts + + # FROM '/search?{query}' + if parts[1] == 'search' && parts.size == 2 + # TO '/api/records/{query}' + "https://#{uri.host}/api/records?#{uri.query}" + # FROM '/record/{recid}' + elsif parts[1] == 'record' && parts.size == 3 + # TO 'api/record/{recid}' + "https://#{uri.host}/api/#{parts[1]}/#{parts[2]}" + end + end + + def update_url_field_value(url, field, value) + uri = URI.parse(url) + params = URI.decode_www_form(uri.query || '').to_h + params[field] = value.to_s + uri.query = URI.encode_www_form(params) + uri.to_s + end + + def add_records_material(initial_data) + data = initial_data + + while data + hits = data.dig('hits', 'hits') || [] + hits.each { |hit| add_material to_material(hit) } + + next_url = data.dig('links', 'next') + break unless next_url + + response = open_url(next_url)&.read + data = response ? JSON.parse(response) : nil + end + end + + # Sets material hash keys and values and add them to material + def to_material(data) + metadata = (data['metadata'] || {}).transform_keys { |k| k == '_cds' ? 'cds' : k } + + material = OpenStruct.new + material.title = metadata.dig('title', 'title').to_s.titleize + material.url = "#{CDS_VIDEOS_RECORD_URL}#{data['id']}" + material.description = get_description(metadata) + material.keywords = Array(metadata['keywords']).first(10).map { |k| k['name'] }.compact + material.licence = metadata.dig('copyright', 'holder') == 'CERN' ? 'other-at' : 'notspecified' + material.status = 'Active' + material.contact = get_contact(metadata) + material.version = Array(metadata['report_number']).first + material.date_created = metadata['date'] + material.date_published = metadata['publication_date'] + material.authors = get_authors(metadata) + material.contributors = get_contributors(metadata) + type = metadata['type'] + category = metadata['category'] + resource_type = [type, category].compact.map { |v| v.to_s.titleize }.join(' – ') + material.resource_type = resource_type.present? ? [resource_type] : [] + material + end + + def get_description(metadata) + description = metadata['description'].split('
')[0] + "\n\n" + description = description&.<< "Video duration: #{metadata['duration']}\n\n" if metadata['duration'] + description = description + metadata['additional_descriptions']&.map { |k| k['description'] }&.join("\n\n") + "\n\n" || '' if metadata['additional_descriptions'] + description = description&.<< "Related identifier:\n- Is part of: #{metadata['alternate_identifiers'][0]['value']} (URL)\n\n" if metadata['alternate_identifiers']&.any? + description = description&.<< "Copyright: [#{metadata.dig('copyright', 'holder')}](#{metadata.dig('copyright', 'url')}) (#{metadata.dig('copyright', 'year')})" if metadata.dig('copyright', 'holder') + description + end + + def get_authors(metadata) + metadata['contributors'] + &.select { |c| c['role'] == 'Creator' } + &.reject { |c| c['name'].downcase == 'cern' } + &.map { |c| c['name']&.gsub(',', '')&.titleize } || '' + end + + def get_contributors(metadata) + metadata['contributors'] + &.reject { |c| c['role'] == 'Creator' || c['role'] == 'ContactPerson' } + &.reject { |c| c['name'].downcase == 'cern' } + &.map { |c| c['name']&.gsub(',', '')&.titleize } || '' + end + + def get_contact(metadata) + contributors = metadata['contributors'] || [] + + # 1. Check for ContactPerson name + contact_person = contributors.find { |c| c['role'] == 'ContactPerson' } + contact_name = contact_person&.[]('name')&.gsub(',', '')&.titleize + + # 2. Check for Creator email + creator = contributors.find { |c| c['role'] == 'Creator' } + creator_email = creator&.[]('email')&.gsub(',', '')&.titleize + + # 3. Check for any other contributor email + other_contributor = contributors.find { |c| !%w[ContactPerson Creator].include?(c['role']) } + other_email = other_contributor&.[]('email') + + contact_name.presence || creator_email.presence || other_email || '' + end + end + end +end diff --git a/lib/ingestors/heptraining/epic_tutorials_ingestor.rb b/lib/ingestors/heptraining/epic_tutorials_ingestor.rb new file mode 100644 index 000000000..1476ef3db --- /dev/null +++ b/lib/ingestors/heptraining/epic_tutorials_ingestor.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'yaml' + +module Ingestors + module Heptraining + class EpicTutorialsIngestor < Ingestor + def self.config + { + key: 'epic_tutorials', + title: 'ePIC Tutorials', + category: :materials, + user_agent: 'TeSS ePIC Tutorials ingestor' + } + end + + def read(url) + @verbose = false + response = open_url(url, raise: true) + return unless response + + process_frontmatter(response.read) + rescue StandardError => e + Rails.logger.error("#{e.class}: read() failed, #{e.message}") + end + + private + + def process_frontmatter(content) + yaml_match = content.match(/\A---\s*\n(.*?)\n?---\s*/m) + raise 'No YAML frontmatter found' unless yaml_match + + data = YAML.safe_load(yaml_match[1], permitted_classes: [Date, Time]) + tutorials = data['tutorials'] || [] + + tutorials.each do |tutorial| + process_tutorial(tutorial) + end + end + + def process_tutorial(tutorial) + url = tutorial['url'] + return if url.blank? + + materials, events = fetch_from_other_ingestor(url) + + # Build external resources list from YAML + ext_resources = (tutorial['resources'] || []).map do |res| + next if res['url'].blank? + + kind = res['kind'].present? ? res['kind'].capitalize : nil + title = res['label'].present? ? "#{kind} #{res['label']}" : kind + { title: title, url: res['url'] } + end.compact + + # Attach external resources to returned materials + (materials + events).each do |material| + material.external_resources ||= [] + material.external_resources.concat(ext_resources) + end + + @materials.concat(materials) + @events.concat(events) + end + + def fetch_from_other_ingestor(url) + ingestor = if github_url?(url) + Ingestors::GithubIngestor.new + elsif indico_url?(url) + Ingestors::IndicoIngestor.new + end + + return [[], []] unless ingestor + + ingestor.read(url) + [ingestor.materials, ingestor.events] + end + + def github_url?(url) + uri = URI.parse(url) + uri.host&.downcase == 'github.com' || uri.host&.downcase&.end_with?('.github.io') + end + + def indico_url?(url) + url.include?('indico') + end + end + end +end \ No newline at end of file diff --git a/lib/ingestors/ingestor_factory.rb b/lib/ingestors/ingestor_factory.rb index d4c2d3fac..97e8c9cdb 100644 --- a/lib/ingestors/ingestor_factory.rb +++ b/lib/ingestors/ingestor_factory.rb @@ -15,8 +15,7 @@ def self.ingestors Ingestors::GithubIngestor, Ingestors::EventRSSIngestor, Ingestors::MaterialRSSIngestor, - Ingestors::YoutubeIngestor, - Ingestors::CdsVideosIngestor + Ingestors::YoutubeIngestor ] + taxila_ingestors + llm_ingestors + heptraining_ingestors end @@ -57,7 +56,9 @@ def self.llm_ingestors def self.heptraining_ingestors [ - Ingestors::Heptraining::GrayScottIngestor + Ingestors::Heptraining::GrayScottIngestor, + Ingestors::Heptraining::CdsVideosIngestor, + Ingestors::Heptraining::EpicTutorialsIngestor ] end diff --git a/test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md b/test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md new file mode 100644 index 000000000..56aabb26f --- /dev/null +++ b/test/fixtures/files/ingestion/heptraining/epictutorials/epic-page.md @@ -0,0 +1,35 @@ +--- +title: ePIC Tutorials +description: Tutorials for ePIC environment setup, analysis, simulation, reconstruction, file access, and benchmarks. +name: tutorials +layout: default +tutorials: + + - title: "Setting up an environment" + url: "https://github.com/hsf-training/cpluspluscourse" + summary: "How to get eic-shell and what it is
Usage of github" + image: "/assets/images/tutorials/tutorial-eicshell.png" + resources: + - label: "1" + url: "https://www.youtube.com/watch?v=Y0Mg24XLomY" + kind: "youtube" + - label: "2" + url: "https://www.youtube.com/watch?v=5HmzFnYW4W4" + kind: "youtube" + + - title: "Reconstruction framework" + url: "https://github.com/hsf-training/cpluspluscourse" + summary: "eicrecon in a nutshell.
JANA2 framework , Plugins, Factories, and Algorithms" + image: "/assets/images/tutorials/tutorial-jana2.png" + + - title: "Getting started with physics analysis" + url: "https://indico.bnl.gov/event/27123" + summary: "Understanding EPIC detector regions
Quality assurance plots for physics
Acceptance and efficiency
Beam effects and afterburner
Full-chain analysis" + image: "/assets/images/tutorials/tutorial-physics-analysis.png" + resources: + - url: "https://youtu.be/goONxXudL-s" + kind: "youtube" +--- + +
+
\ No newline at end of file diff --git a/test/unit/ingestors/cds_videos_ingestor_test.rb b/test/unit/ingestors/heptraining/cds_videos_ingestor_test.rb similarity index 97% rename from test/unit/ingestors/cds_videos_ingestor_test.rb rename to test/unit/ingestors/heptraining/cds_videos_ingestor_test.rb index a428948ae..0f275202f 100644 --- a/test/unit/ingestors/cds_videos_ingestor_test.rb +++ b/test/unit/ingestors/heptraining/cds_videos_ingestor_test.rb @@ -4,14 +4,14 @@ class CdsVideosIngestorTest < ActiveSupport::TestCase setup do - @ingestor = Ingestors::CdsVideosIngestor.new + @ingestor = Ingestors::Heptraining::CdsVideosIngestor.new # API record webmock('https://videos.cern.ch/api/record/3023615', 'cdsvideos/cds.ingestor.json') end test 'returns expected ingestor config' do - config = Ingestors::CdsVideosIngestor.config + config = Ingestors::Heptraining::CdsVideosIngestor.config assert_equal 'cds videos', config[:key] assert_equal :materials, config[:category] end diff --git a/test/unit/ingestors/heptraining/epic_tutorials_ingestor_test.rb b/test/unit/ingestors/heptraining/epic_tutorials_ingestor_test.rb new file mode 100644 index 000000000..246d4842e --- /dev/null +++ b/test/unit/ingestors/heptraining/epic_tutorials_ingestor_test.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +require 'test_helper' + +class EpicTutorialsIngestorTest < ActiveSupport::TestCase + setup do + @ingestor = Ingestors::Heptraining::EpicTutorialsIngestor.new + @user = users(:regular_user) + @content_provider = content_providers(:portal_provider) + + webmock('https://raw.githubusercontent.com/eic/tutorials/main/tutorials.md', 'heptraining/epictutorials/epic-page.md') + # Github + webmock('https://github.com/hsf-training/cpluspluscourse', 'github/mock.html') + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse', 'github/api-github-com.json') + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse/contributors', 'github/contributors.json') + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse/contents/README.md', 'github/readme.json') + webmock('https://api.github.com/repos/hsf-training/cpluspluscourse/releases', 'github/releases.json') + + # Indico + webmock('https://indico.bnl.gov/event/27123', 'indico/indico.html') + webmock('https://indico.bnl.gov/event/27123/event.ics', 'indico/event.ics') + end + + test 'config' do + config = Ingestors::Heptraining::EpicTutorialsIngestor.config + + assert_equal 'epic_tutorials', config[:key] + assert_equal 'ePIC Tutorials', config[:title] + assert_equal :materials, config[:category] + assert_equal 'TeSS ePIC Tutorials ingestor', config[:user_agent] + end + + test 'github_url?' do + assert @ingestor.send(:github_url?, 'https://github.com/eic/tutorial-setting-up-environment') + assert @ingestor.send(:github_url?, 'https://eic.github.io/tutorial-setting-up-environment') + refute @ingestor.send(:github_url?, 'https://indico.bnl.gov/event/27123/') + refute @ingestor.send(:github_url?, 'not_a_valid_url') + end + + test 'indico_url?' do + assert @ingestor.send(:indico_url?, 'https://indico.bnl.gov/event/27123/') + assert @ingestor.send(:indico_url?, 'https://indico.cern.ch/event/12345/') + refute @ingestor.send(:indico_url?, 'https://eic.github.io/scavenger-hunt') + end + + test 'should read epic tutorials source and attach external resources' do + # Stub delegation to sub-ingestors + @ingestor.read('https://raw.githubusercontent.com/eic/tutorials/main/tutorials.md') + @ingestor.write(@user, @content_provider) + + assert_equal 2, @ingestor.materials.count + assert_equal 1, @ingestor.events.count + + evt = @ingestor.events.detect { |e| e.title == '14th HEP C++ Course and Hands-on Training - The Essentials' } + assert evt.persisted? + assert_equal evt.url, 'https://indico.cern.ch/event/1617123/' + assert_equal evt.description, 'speakers and Zoom here, however it is not hybrid' + assert_equal 1, evt.external_resources.size + extres0 = evt.external_resources[0] + assert_equal 'Youtube', extres0.title + assert_equal 'https://youtu.be/goONxXudL-s', extres0.url + + + mat = @ingestor.materials.detect { |e| e.title == 'Cpluspluscourse' } + assert mat.persisted? + assert_equal mat.description, 'C++ Course Taught at CERN' + assert_equal 2, mat.external_resources.size + extres1 = mat.external_resources[0] + assert_equal 'Youtube 1', extres1.title + assert_equal 'https://www.youtube.com/watch?v=Y0Mg24XLomY', extres1.url + extres2 = mat.external_resources[1] + assert_equal 'Youtube 2', extres2.title + assert_equal 'https://www.youtube.com/watch?v=5HmzFnYW4W4', extres2.url + end + + test 'fetch_from_other_ingestor delegates to GithubIngestor and IndicoIngestor' do + gh_url = 'https://eic.github.io/tutorial-setting-up-environment' + indico_url = 'https://indico.bnl.gov/event/27123/' + unknown_url = 'https://example.com/tutorial' + + mock_gh_material = Material.new(title: 'GH Mat') + mock_indico_event = Event.new(title: 'Indico Evt') + + # Test GitHub delegation + gh_mock = Minitest::Mock.new + gh_mock.expect(:read, true, [gh_url]) + gh_mock.expect(:materials, [mock_gh_material]) + gh_mock.expect(:events, []) + + Ingestors::GithubIngestor.stub(:new, gh_mock) do + gh_res = @ingestor.send(:fetch_from_other_ingestor, gh_url) + assert_equal [mock_gh_material], gh_res[0] + assert_equal [], gh_res[1] + end + gh_mock.verify + + # Test Indico delegation + indico_mock = Minitest::Mock.new + indico_mock.expect(:read, true, [indico_url]) + indico_mock.expect(:materials, []) + indico_mock.expect(:events, [mock_indico_event]) + + Ingestors::IndicoIngestor.stub(:new, indico_mock) do + indico_res = @ingestor.send(:fetch_from_other_ingestor, indico_url) + assert_equal [], indico_res[0] + assert_equal [mock_indico_event], indico_res[1] + end + indico_mock.verify + + # Test unknown URL + unknown_res = @ingestor.send(:fetch_from_other_ingestor, unknown_url) + assert_equal [[], []], unknown_res + end + + test 'std errors when exception is raised during read' do + @ingestor.stub(:open_url, ->(*) { raise StandardError, 'connection error' }) do + mock_logger = Minitest::Mock.new + mock_logger.expect(:error, nil, ['StandardError: read() failed, connection error']) + + Rails.stub(:logger, mock_logger) do + @ingestor.read('https://raw.githubusercontent.com/eic/tutorials/main/tutorials.md') + end + + mock_logger.verify + end + end + + private + + def webmock(url, filename) + file = Rails.root.join('test', 'fixtures', 'files', 'ingestion', filename) + WebMock.stub_request(:get, url).to_return(status: 200, headers: {}, body: file.read) + end +end \ No newline at end of file