Artifact f10669f8a12e371823cd82a5731228d0fe2657baa55db16b9f0564ce3d37f83f:
- File
stackexchange-favs-to-pinboard.rb
— part of check-in
[09c6947036]
at
2017-03-12 14:54:59
on branch master
— Make this actually work for lots of favs
Don't know if somehting changed or this was just always semi-broken
(probable), but noticed today it wasn't adding any new favs when I knew
for a fact I had some; Perhaps, also, I have a lot more favs than when I
first wrote this.Previously this assumed that the response returned all favs for a site
whereas actually by default it returns just 30. Perhaps by luck I'd been
picking up new ones because of using "sort=activity"; a strange choice
on it's own as if I'd used "sort=added" and run this fairly regularly
perhaps I'd never had noticed any issue? Anyway, now it does the correct
thing which is pull 100 at a time and look for the has_more parameter in
what is returned so it knows whether it needs to fetch another page.This does mean get_favs now returns an array of items (makes more sense)
than the actual encompassing response as it used to. (user: base@atomicules.co.uk size: 4010)
#StackExchange Favs to Pinboard require 'open-uri' require 'json' require 'optparse' require 'cgi' require 'logger' optparse = OptionParser.new do |opts| opts.on('-i', '--id ID', "Stackexchange ID") { |i| StackID = i } opts.on('-u', '--user USER', "Pinboard username") { |u| User = u } opts.on('-t', '--token TOKEN', "Pinboard API Token") { |t| Token = t } end optparse.parse! def get_sites(id) response = open("https://api.stackexchange.com/2.2/users/#{id}/associated") parsed = parse(response.read) end def parse(response_string) #Seems like Ruby automatically handles gzip compression now. No need to decompress. parsed = JSON.parse(response_string) end def get_favs(site, id) puts "Fetching favs for #{site}" has_more = true items = [] page = 1 while has_more response = open("https://api.stackexchange.com/2.2/users/#{id}/favorites?order=desc&sort=added&pageSize=100&page=#{page}&site=#{site}") #Use response.read not .string as some returns are large enough to go to a temp file parsed = parse(response.read) items += parsed["items"] has_more = parsed["has_more"] page += 1 end puts "#{items.length} favs" items end class Pinboard @@logger = Logger.new(STDOUT) @@logger.level = Logger::INFO @@rate_limit = 3 Api_url = "https://api.pinboard.in/v1/" def initialize(user, token) @user = user @token = token end def add(url, description, extended=nil, tags=nil, replace="no") attempts = 1 posted = false #At minimum must have url and description array_parameters = "&url=#{CGI.escape(url)}&description=#{CGI.escape(description)}" #Could loop through the below unless extended.nil? array_parameters += "&extended=#{CGI.escape(extended)}" end unless tags.nil? #TODO: Need to check whether tags_escaped will work array_parameters += "&tags=#{CGI.escape(tags)}" end until (@@rate_limit > 60) | (attempts > 3) | posted response = open("#{Api_url}posts/add?auth_token=#{@user}:#{@token}"+array_parameters+"&replace=#{replace}&format=json") @@logger.debug(response.string) @@logger.debug(response.status) response_json = JSON.parse(response.string) if (response.status[0] == "200") & (response_json["result_code"] == "done") @@logger.info("Added #{url}") posted = true elsif (response.status[0] == "200") & (response_json["result_code"] == "item already exists") @@logger.info("Skipping #{url}, already exists") posted = true elsif response.status[0] == "429" # 429 Too Many Requests, increase rate limit @@rate_limit *= 2 @@logger.warn("Rate Limit increased to #{$rate_limit} seconds") end attempts += 1 #Rate limit as per Pinboard API requirements sleep @@rate_limit end if @@rate_limit > 60 @@logger.error("Rate limit has exceeded 60 secs, let's try again another time") elsif attempts > 3 @@logger.error("Failed 3 times to save #{url}, bombing out") end posted end end if defined?(StackID) and defined?(User) and defined?(Token) cache_file = ENV["HOME"]+"/.stackexchange_favs_to_pinboard" pb = Pinboard.new(User, Token) if File.exists?(cache_file) cache = JSON.parse(File.read(cache_file)) else cache = {} end parsed = get_sites(StackID) parsed["items"].each do |site| favs = get_favs(site["site_url"].sub("http://", "").sub("https://", "").sub(".stackexchange", "").sub(".com", ""), site["user_id"]) #Don't make more than 30 requests per second sleep (1.0/30) favs.each do |fav| title = fav["title"] tags = fav["tags"] link = fav["link"] #Check cache to see if already added to pinboard unless cache.has_key?(link) #Need to unescape so can re-escape in Pinboard code #Still want no default replace, just in case cache doesn't exist pb.add(link, CGI.unescape_html(title), nil, tags.join(", ")+", stackexchangefavs", "no") #Add to cache. Should check for success really cache[link] = true end end end #Write out cache File.open(cache_file, "w") do |file| file << cache.to_json end end