script to download every page

consultation_ipred/Gemfile

 source "http://rubygems.org"
 
 gem 'nokogiri'
+gem 'selenium-webdriver'

consultation_ipred/Gemfile.lock

 GEM
   remote: http://rubygems.org/
   specs:
+    childprocess (0.2.2)
+      ffi (~> 1.0.6)
+    ffi (1.0.9)
+    json_pure (1.6.1)
     nokogiri (1.5.0)
+    rubyzip (0.9.4)
+    selenium-webdriver (2.7.0)
+      childprocess (>= 0.2.1)
+      ffi (>= 1.0.7)
+      json_pure
+      rubyzip
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   nokogiri
+  selenium-webdriver

consultation_ipred/download.rb

+require "rubygems"
+require "bundler/setup"
+
+require 'selenium-webdriver'
+
+driver = Selenium::WebDriver.for :firefox
+
+["Organisations", "Public authorities"].each do |name|
+  puts name
+  
+  driver.get "https://circabc.europa.eu/w/browse/d7497d8f-5e7e-407e-b682-71d1b71f99a5"
+  wait = Selenium::WebDriver::Wait.new(:timeout => 5)
+
+  link = wait.until {
+    element = driver.find_element(:link_text => name)
+    element if element.displayed?
+  }
+  link.click
+
+  filename = name.downcase.gsub(/\s+/, "_") + ".links"
+  File.open(filename, "w") do |file|
+    
+    loop do
+      pdf_links = wait.until {
+        driver.find_elements(:partial_link_text => ".pdf")
+      }
+
+      file.puts pdf_links.map { |link| link["href"] }.join("\n")
+
+      begin
+        next_link = driver.find_element(:css => "a[title=\"Next Page\"]")
+      rescue Selenium::WebDriver::Error::NoSuchElementError
+        break
+      end
+      next_link.click
+    end
+  end
+end
+
+driver.quit

consultation_ipred/public_authorities.links

+https://circabc.europa.eu/d/d/workspace/SpacesStore/39852257-57a4-4002-a6df-8b799661c3ea/ak_oesterreich_de.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/32fe9217-6c5a-4b3b-a954-88e491337f7f/belgium_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/3346b4d7-2171-428d-a9af-fe31a3dd67c9/bulgaria_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/2c8a3544-c150-4fc0-941b-6496375682b8/czech_republic_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/295fb523-cbec-4f7f-b9f1-55ec2715652b/danish_chamber_of_commerce_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/f37efa48-d5c4-4b22-8d87-893d50170a70/denmark_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/82d024ea-ce88-4974-9122-2bf368f91906/deutscher_bundestag_de.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/fa293e2c-cfae-43ed-abcb-59ac2d31a4fb/european_parliament_committee_on_legal_affairs_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/cce241c1-0810-4d19-b288-5c5f6d8e26d4/finland_ministry_of_empl_and_economy_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/13b94e78-5719-4892-af14-469f47d8cf2d/finnish_commerce_federation_fi.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/4edaf82a-d143-4a07-9880-e59529afc595/france.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/0ef00dda-f4be-4950-bfad-f48baf11cb0c/germany_de.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/c2cdc614-5073-4306-8293-2c40c9fa84da/hadopi_fr.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/7668b8f8-3080-4de2-8a90-62c7d6be0e60/hungary_ministry_public_%20administration_and_justice_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/ddc21a57-b983-4ede-a02f-59dd3f2bc8a6/ireland_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/0c470e92-6585-442f-9d26-e76046744085/italy_ministry_of_agrifood_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/71f09778-f7f6-4b49-af15-3dd02d03c42e/italy_office_of_the_prime_minister_it.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/0f084a74-d6ca-41e2-afe5-97b74f539ea3/latvia_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/d941e7ac-0ad0-4934-b05d-711f816b95e7/lithuania_ministry_culture_lt.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/c8884ac2-d625-44e5-a3a7-7e7b98b111ac/malta_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/607ebe73-a7ae-489e-8f14-2dff48521320/netherlands_ministry_security_justice_annex1_nl.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/36934912-7ec6-4f14-aa80-e8780fa56cb0/netherlands_ministry_security_justice_annex2_nl.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/11bc6214-9b28-4d89-a1f6-58bf0a4e1abd/netherlands_ministry_security_justice_nl.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/54f046e5-c3b3-4b75-8394-0f84d204ae97/parti_pirate_fr.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/e4d28106-09a7-4989-8aad-cfbe36fdfc02/poland_ministry_culture_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/b32505cb-63b5-4828-a4dd-81ba03fe8a43/poland_ministry_culture_pl.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/fcf00a0f-7d50-4423-8101-2deb638cd16e/portugal_pt.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/919f019f-bb07-443d-801c-f6db3a668134/romania_en.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/046d1325-6cc6-4f8e-aee5-25e354b60f69/romania_ro.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/05142029-2e42-43d6-b4b8-0bb42d5e836a/slovakia_sk.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/8c1ac475-47f7-486f-adc6-d68fddd7a141/spain_ministry_of_justice_es.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/6436e9a1-1ba6-414e-af8c-d50fa3ac2c52/uk_governement.pdf
+https://circabc.europa.eu/d/d/workspace/SpacesStore/df9bf458-a928-43c2-931e-fe722ca1e317/wko_austria_de.pdf