Commit 161205a1 authored by Michael Witrant's avatar Michael Witrant

script to download every page

parent 9e3cae2b
source "http://rubygems.org"
gem 'nokogiri'
gem 'selenium-webdriver'
GEM
remote: http://rubygems.org/
specs:
childprocess (0.2.2)
ffi (~> 1.0.6)
ffi (1.0.9)
json_pure (1.6.1)
nokogiri (1.5.0)
rubyzip (0.9.4)
selenium-webdriver (2.7.0)
childprocess (>= 0.2.1)
ffi (>= 1.0.7)
json_pure
rubyzip
PLATFORMS
ruby
DEPENDENCIES
nokogiri
selenium-webdriver
require "rubygems"
require "bundler/setup"
require 'selenium-webdriver'
driver = Selenium::WebDriver.for :firefox
["Organisations", "Public authorities"].each do |name|
puts name
driver.get "https://circabc.europa.eu/w/browse/d7497d8f-5e7e-407e-b682-71d1b71f99a5"
wait = Selenium::WebDriver::Wait.new(:timeout => 5)
link = wait.until {
element = driver.find_element(:link_text => name)
element if element.displayed?
}
link.click
filename = name.downcase.gsub(/\s+/, "_") + ".links"
File.open(filename, "w") do |file|
loop do
pdf_links = wait.until {
driver.find_elements(:partial_link_text => ".pdf")
}
file.puts pdf_links.map { |link| link["href"] }.join("\n")
begin
next_link = driver.find_element(:css => "a[title=\"Next Page\"]")
rescue Selenium::WebDriver::Error::NoSuchElementError
break
end
next_link.click
end
end
end
driver.quit
This diff is collapsed.
https://circabc.europa.eu/d/d/workspace/SpacesStore/39852257-57a4-4002-a6df-8b799661c3ea/ak_oesterreich_de.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/32fe9217-6c5a-4b3b-a954-88e491337f7f/belgium_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/3346b4d7-2171-428d-a9af-fe31a3dd67c9/bulgaria_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/2c8a3544-c150-4fc0-941b-6496375682b8/czech_republic_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/295fb523-cbec-4f7f-b9f1-55ec2715652b/danish_chamber_of_commerce_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/f37efa48-d5c4-4b22-8d87-893d50170a70/denmark_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/82d024ea-ce88-4974-9122-2bf368f91906/deutscher_bundestag_de.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/fa293e2c-cfae-43ed-abcb-59ac2d31a4fb/european_parliament_committee_on_legal_affairs_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/cce241c1-0810-4d19-b288-5c5f6d8e26d4/finland_ministry_of_empl_and_economy_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/13b94e78-5719-4892-af14-469f47d8cf2d/finnish_commerce_federation_fi.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/4edaf82a-d143-4a07-9880-e59529afc595/france.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/0ef00dda-f4be-4950-bfad-f48baf11cb0c/germany_de.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/c2cdc614-5073-4306-8293-2c40c9fa84da/hadopi_fr.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/7668b8f8-3080-4de2-8a90-62c7d6be0e60/hungary_ministry_public_%20administration_and_justice_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/ddc21a57-b983-4ede-a02f-59dd3f2bc8a6/ireland_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/0c470e92-6585-442f-9d26-e76046744085/italy_ministry_of_agrifood_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/71f09778-f7f6-4b49-af15-3dd02d03c42e/italy_office_of_the_prime_minister_it.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/0f084a74-d6ca-41e2-afe5-97b74f539ea3/latvia_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/d941e7ac-0ad0-4934-b05d-711f816b95e7/lithuania_ministry_culture_lt.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/c8884ac2-d625-44e5-a3a7-7e7b98b111ac/malta_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/607ebe73-a7ae-489e-8f14-2dff48521320/netherlands_ministry_security_justice_annex1_nl.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/36934912-7ec6-4f14-aa80-e8780fa56cb0/netherlands_ministry_security_justice_annex2_nl.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/11bc6214-9b28-4d89-a1f6-58bf0a4e1abd/netherlands_ministry_security_justice_nl.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/54f046e5-c3b3-4b75-8394-0f84d204ae97/parti_pirate_fr.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/e4d28106-09a7-4989-8aad-cfbe36fdfc02/poland_ministry_culture_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/b32505cb-63b5-4828-a4dd-81ba03fe8a43/poland_ministry_culture_pl.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/fcf00a0f-7d50-4423-8101-2deb638cd16e/portugal_pt.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/919f019f-bb07-443d-801c-f6db3a668134/romania_en.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/046d1325-6cc6-4f8e-aee5-25e354b60f69/romania_ro.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/05142029-2e42-43d6-b4b8-0bb42d5e836a/slovakia_sk.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/8c1ac475-47f7-486f-adc6-d68fddd7a141/spain_ministry_of_justice_es.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/6436e9a1-1ba6-414e-af8c-d50fa3ac2c52/uk_governement.pdf
https://circabc.europa.eu/d/d/workspace/SpacesStore/df9bf458-a928-43c2-931e-fe722ca1e317/wko_austria_de.pdf
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment