fetchheise-idxpage-idxpageurls

#!/usr/bin/env ruby
# in: URL of index page on cmdline
# out: URLs of all index pages in that forum/"Beitragsübersicht" on stdout,
#      one URL per line

require 'nokogiri'

url = ARGV[0] or raise "usage: #{$0} <url>"

page = Nokogiri::HTML(IO.popen("wget-withbrowsersettings -q -O- '#{url}'"))
spans = page.css('span').select{|span| span['class'] and span['class'].include?('seiten')}
links = (spans and spans[0].css('a').map{|a| a['href'] })

if links
    # last link is biggest N of all links ending in /page-<N>/.
    re = /^(.*?\/)page-(.*?)\/$/
    base = links[0].match(re)[1]
    max = links.select{|l| l.match(re) }.map{|l| Integer(l.match(re)[2]) }.max
    all_links = (1..max).map{|i| "#{base}page-#{i}/" }
    if url.match(re)
        all_links = (all_links + [url]).uniq
    end
    all_links.each do |l|
        puts l
    end
else
    # no index page links found => url itself is the only index page
    puts url
end