Is there a more user-friendly way to download multiple articles from arXiv?

Question

I am subscribed to the arXiv daily digests for my subject area. I find myself, every day, doing the following, when I get to work:

1) Browsing through the arXiv email and right-clicking on the articles of interest to me (to open each of the pages in a different tab in my browser)

2) For each of these pages, downloading the pdf by clicking on the link, re-naming it <article-title> by <author name(s)>.pdf, and saving the pdf in a directory on my PC

[3) the directory is then automatically synched to my tablet during the day]

4) Looking at the articles on my commute home on my tablet.

I am bored of doing (2). It only takes 10-20 seconds per article (depending on things like how many carriage returns or inappropriate characters I have to remove from the cut-and-pasted article title and author names -- e.g. math characters (I am a mathematician and symbols in titles do not cut and paste well)) but when I'm interested in 5 articles and one has a lengthy title with carriage returns and symbols in, my mind wanders and I start thinking about whether this has already been automated by someone, because it seems to me that there would be no obstruction to doing so in theory, but I would not be capable of doing it myself. Does such an automation exist?

Answer 1

There you go!

Take any of the new or recent links from https://arxiv.org/ and substitute it under Settings.

#!/usr/bin/python3
# encoding=utf8

import os, re, subprocess, sys
import urllib.request as urllib2
import urllib.parse
from bs4 import BeautifulSoup

version = 1.0

arguments = {}
arguments['-h, --help'] = 'Print help'
arguments['-v, --version'] = 'Print Version'

# ================== Settings ====================

url = "https://arxiv.org/list/astro-ph/new"

# ================================================

class color:
   PURPLE = '\033[95m'
   CYAN = '\033[96m'
   DARKCYAN = '\033[36m'
   BLUE = '\033[94m'
   GREEN = '\033[92m'
   YELLOW = '\033[93m'
   RED = '\033[91m'
   BOLD = '\033[1m'
   UNDERLINE = '\033[4m'
   END = '\033[0m'

if __name__ == "__main__":

  # =============== Argument parser=================

  if any([1 if arg in sys.argv else 0 for arg in ['-v', '--version']]):
      print(version)
      sys.exit(0)

  if any([1 if arg in sys.argv else 0 for arg in ['-h', '--help']]):

    name = os.path.basename(sys.argv[0])

    # Display help
    print("This is {program}. Get your daily arXiv-dose.\n".format(program=name))
    print("Usage: ./{program}".format(program=name))
    print("Currently I'm fetching", url, '\n')

    for key in arguments:
        print("\t{:15}: {}".format(key, arguments[key]))

    sys.exit(0)

  # ================================================

  # ============ Generate and fetch url ============


  try:
    req = urllib2.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
    html = urllib2.urlopen(req)

  except urllib2.HTTPError:
    print(url)
    print('"{}" not found. Correct spelling?'.format(search))
    sys.exit(0)

  # ================================================

  # ================= Find papers ==================

  soup = BeautifulSoup(html, "lxml")

  articles = {}

  # Get DOI and URL
  papers = soup.find_all("dt")

  for c, nnn in zip( papers, range( len(papers) ) ):

    articles[nnn] = {}

    doi = c.find_all("a", title="Abstract")[0]
    doi = doi.get_text()
    articles[nnn]["doi"] = doi

    link = c.find_all("a", title="Download PDF")[0].get("href")
    articles[nnn]["url"] = 'https://arxiv.org' + link

  # Get Title, Authors and Abstract
  meta = soup.find_all("div", class_="meta")

  for c, nnn in zip(meta, range(len(meta))):

    title = c.find("div", class_="list-title")
    title = title.get_text().replace('Title: ','')
    articles[nnn]["title"] = title.strip()

    authors = c.find("div", class_="list-authors")
    authors = authors.get_text().replace('Authors:','').replace('\n','')
    authors = re.sub('[a-zA-Z]+\.+\ ','',authors)
    articles[nnn]["authors"] = authors.strip()

    try:
      abstract = c.find("p", class_="mathjax").get_text().replace('\n',' ')
    except AttributeError:
      pass

    articles[nnn]["abstract"] = abstract

  # List findings
  for paper in articles.keys():

    print( '\n' + color.BOLD + color.UNDERLINE +'{:5}'.format(paper) + color.END,
           articles[paper]["title"])
    print( 6 * ' ' + articles[paper]["authors"], '\n' )
    print( ' ' + articles[paper]["abstract"] )

  # Get user input list
  while True:

    download = input( '\n' + color.BOLD + 'Download (2 12 ..): ' + color.END )

    try:
      download = [ int(i) for i in download.split() ]
      break

    except ValueError:
      print('Not a valid list: "{}"'.format(download))
      pass

  for file in download:

    url = articles[file]["url"]
    filename = '{}-{}-{}.pdf'.format(articles[file]["title"], articles[file]["authors"], articles[file]["doi"])

    # EXT4 limits filenames to 255 characters

    if len(filename) > 254:

      filename = articles[file]["title"] + '-'

      for author in articles[file]["authors"].split():
        if len(author) + len(filename) + len(articles[file]["doi"]) + 5 < 255:
          filename += author.strip()

      filename = filename[:-1] + '-' + articles[file]["doi"] + ".pdf"
      print(color.BOLD + 'Warning:' + color.END + 'Too many authors for |filename| < 256.')
      print('Truncating to ', filename)

    # Download
    subprocess.call(["wget", '--quiet', '--show-progress', '--header', "User-Agent: Mozilla/5.0", "--output-document", '{}'.format(filename), url])

This will give you a complete list with title, author and abstract. You can then enter a list of numbers to download as {title}-{authors}-{doi}.pdf.

[...]

  120 Flavours in the box of chocolates: chemical abundances of kinematic  substructures in the nearby stellar halo
      Jovan Veljanoski, Amina Helmi 

 Different subtleties and problems associated with a nonrelativistic limit of the field theory to the Schroedinger theory are discussed. In this paper, we revisit different cases of the nonrelativistic limit of a real and complex scalar field in the level of the Lagrangian and the equation of motion. We develop the nonrelativistic limit of the Dirac equation and action in the way that the nonrelativistic limit of spin-$\frac{1}{2}$ wave functions of particles and antiparticles appear simultaneously. We study the effect of a potential like $U(\phi)\propto \phi^4$ which can be attributed to axion dark matter field in this limit. We develop a formalism for studying the nonrelativistic limit of antiparticles in the quantum mechanics. We discussed the non-local approach for the nonrelativistic limit and its problems. 

  121 The Masses and Accretion Rates of White Dwarfs in Classical and  Recurrent Novae
      Michael Shara, Dina Prialnik, Yael Hillman, Attay Kovetz 

 Different subtleties and problems [...]

Download (2 12 ..):

---

Usage

E.g.:

1. Save the script as arxiv in /usr/local/bin
2. chmod +x /usr/local/bin/arxiv

You should now be able to execute it by just typing arxiv in terminal.

Requirements

• python3
• python-beautifulsoup4 (pip install bs4 if you use python-pip package)
• wget

---

EDIT Current version is available here.

Answer 2

You can use JabRef for this. More precisely, there are plugins by Christoph Lehner that do the job:

• arxiv-rss to browse the list of new preprints and import the ones you want;
• localcopy for downloading the PDF and automatically renaming it (according to a pattern you can define).

This isn't 100% automated (you still need to manually click "download arXiv PDF" once you've imported the entry in your bib file), but this is still much better than doing it all by hand.

Answer 3

IMO, the way to implement this would be as a browser or e-mail client extension. Personally, I have a subscription for e-mail alerts because it lets me select subject areas. Recently I looked into extensions to make that more easily readable, so I searched the Firefox and Thunderbird extensions for things related to arXiv, but my search turned up nothing interesting.

Of course, it could exist e.g. as an extension for some other browser; you know, it is tough to prove nonexistance.

Given the arXiv ID, an “external“ solution should be pretty easy (getting the PDF is a simple manner of wget http://arxiv.org/pdf/$ID, and extracting the title and authors from the abs/$ID page should be simple enough, as well). However, the question is what you gain by that.

Instead, I would recommend Zotero as an alternative type of solution. It lets you save article metadata from arXiv and many other sources in the click of a button. It can also download and archive PDFs automatically, on your PC or on their server (where you get a very limited amount of space for free and can pay to get more).

Answer 4

The following script uses arxiv python API to download friendly named files.

Features:

• Inputs: URLs or .pdf files.
• Outputs: corrected .pdf files.
• Corrects .pdf files that have already been downloaded!

Script:

import os, re, sys, arxiv

def fix_title(title: str) -> str:
    title = " ".join(str.strip(x) for x in title.split("\n"))
    title = title.replace(":", "_")
    return title

def paper_to_filename(paper: arxiv.Result) -> str:
    authors = paper.authors
    author_str = authors[0] if len(authors) == 1 else f"{authors[0]} et al."
    return f"{author_str} - {fix_title(paper.title)}.pdf"

def parse_line(line: str):
    id_pattern = r"(\d{4}\.\d{4,6}(v\d+)?)"
    m = re.match(fr".*{id_pattern}(\.pdf)?$", line)
    return m.group(1) if m is not None else None

paper_ids = [parse_line(line.strip()) for line in sys.stdin.readlines()]
paper_ids = [x for x in paper_ids if x is not None]
papers = arxiv.Search(id_list=paper_ids).results()

for paper, paper_id in zip(papers, paper_ids):
    src_filename = f"{paper_id}.pdf"
    dst_filename = paper_to_filename(paper)
    if os.path.exists(src_filename):
        print(f"[Rename] {src_filename}")
        os.rename(src_filename, dst_filename)
    else:
        print("[Download]")
        paper.download_pdf(filename=dst_filename)
    print(f"file:    {dst_filename}")
    print(f"url:     {paper.entry_id}")
    print(f"authors: {[str(x) for x in paper.authors]}")
    print(f"title:   {paper.title}\n")

Dependencies:

pip install arxiv==1.4.2

Example 1:

Automatically correct all PDF filenames inside a directory:

λ ls
1506.02640.pdf

λ ls *.pdf | python arxiv_downloader.py
[Rename] 1506.02640.pdf
file:    Joseph Redmon et al. - You Only Look Once_ Unified, Real-Time Object Detection.pdf
url:     http://arxiv.org/abs/1506.02640v5
authors: ['Joseph Redmon', 'Santosh Divvala', 'Ross Girshick', 'Ali Farhadi']
title:   You Only Look Once: Unified, Real-Time Object Detection

λ ls
Joseph Redmon et al. - You Only Look Once_ Unified, Real-Time Object Detection.pdf

Example 2:

Manually provide your own list of URLs, filenames, or paper IDs:

λ wget -nc https://arxiv.org/pdf/1506.02640.pdf
λ wget -nc https://arxiv.org/pdf/1502.03167v3.pdf

λ echo "https://arxiv.org/abs/2002.00157
        https://arxiv.org/pdf/1805.11604
        1506.02640.pdf
        1502.03167v3" | python arxiv_downloader.py

[Download]
file:    Mateen Ulhaq et al. - Shared Mobile-Cloud Inference for Collaborative Intelligence.pdf
url:     http://arxiv.org/abs/2002.00157v1
authors: ['Mateen Ulhaq', 'Ivan V. Bajić']
title:   Shared Mobile-Cloud Inference for Collaborative Intelligence

[Download]
file:    Shibani Santurkar et al. - How Does Batch Normalization Help Optimization?.pdf
url:     http://arxiv.org/abs/1805.11604v5
authors: ['Shibani Santurkar', 'Dimitris Tsipras', 'Andrew Ilyas', 'Aleksander Madry']
title:   How Does Batch Normalization Help Optimization?

[Rename] 1506.02640.pdf
file:    Joseph Redmon et al. - You Only Look Once_ Unified, Real-Time Object Detection.pdf
url:     http://arxiv.org/abs/1506.02640v5
authors: ['Joseph Redmon', 'Santosh Divvala', 'Ross Girshick', 'Ali Farhadi']
title:   You Only Look Once: Unified, Real-Time Object Detection

[Rename] 1502.03167v3.pdf
file:    Sergey Ioffe et al. - Batch Normalization_ Accelerating Deep Network Training by Reducing Internal Covariate Shift.pdf
url:     http://arxiv.org/abs/1502.03167v3
authors: ['Sergey Ioffe', 'Christian Szegedy']
title:   Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift

Answer 5

Try Zotero which can save multiple items from certain Web pages (you'd have to view the list in a browser).

Answer 6

FWIW, my Python (2) code that downloads arXiv preprints (PDF and source) given a text file containing hyperlinks (usually, a copypasted arXiv digest from an email):

# batch-download arXiv preprints linked in a text file
# (meant for the emails that come from arXiv).
# Written for use in Cygwin or Linux; not sure how it
# behaves on a normal Windows python.
#
# Syntax:
# - "python arxdown.py mail.txt [folder]":
#   Downloads all arXiv preprints hyperlinked in "mail.txt"
#   into folder [folder].
# - "python arxdown.py https://arxiv.org/abs/1308.0047 [folder]":
#   Downloads https://arxiv.org/abs/1308.0047 into folder [folder].
# If [folder] is not specified, a default one is used.
# Even if the arXiv hyperlink comes with a version number,
# the script downloads the newest version by default; this
# behavior can be disabled with the "-u" switch.

import os
import urllib
import urllib2
import re
import time
import sys
import socket
import shutil
import string
import lxml.html
from unidecode import unidecode
import itertools

defaultpath = "/home/arxiv" # The path into which the downloads should go if no folder was specified.

arxivprefix = "http://arxiv.org" # Replace by one of the mirrors ( https://arxiv.org/help/mirrors ) if the main site is slow/down.

resting_time = 4 # time (in second) to wait between downloads; too small a number seems to get me banned.

args = sys.argv

if "-u" in args:
    # use version numbers provided
    newest = False
    args.remove("-u")
else:
    newest = True

if len(args) > 1:
    # args[1] may be either a file containing URLs, or a URL itself.
    try:
        # Is it a file?
        mail = open(args[1])
        proper_mail = True
    except IOError:
        # Nah.
        mail = [args[1]]
        proper_mail = False
    if len(args) > 2:
        # Whatever remains better be a path.
        tempdirname = args[2]
    else:
        tempdirname = defaultpath
else:
    print "no mail text or hyperlink given"
    sys.exit()

# create temporary folder for downloading, if not already existing.
try:
    os.mkdir(tempdirname)
except OSError:
    pass
os.chdir(tempdirname)

for line in mail:
    if "://arxiv.org/abs/" in line:
        # Which preprint to download?
        for arxid in line.split("://arxiv.org/abs/")[1:]:
            arxid = arxid.split(" ")[0].split("v")
            if len(arxid) > 1:
                arxid, vernum = arxid[:2]
            else:
                arxid = arxid[0]
                vernum = False
            arxid = arxid.strip()
            response = urllib2.urlopen(arxivprefix + "/abs/" + arxid)
            html = response.read().split("\n")
            # Which version to download?
            if (not newest) and vernum:
                vernum = "".join(itertools.takewhile(str.isdigit, vernum))
            else:
                for htmlline in html:
                    if "tablecell arxividv" in htmlline:
                        vernum = htmlline.split(arxid + "v")[1]
                        vernum = vernum.split("\"")[0]
                        break
            arxidv = arxid + "v" + vernum
            print "\n attacking ", arxidv
            # Build filename for the downloads.
            # I am being heavily conservative here; all kinds of
            # harmless symbols get kicked out.
            author_surnames = []
            valid_letters = string.ascii_lowercase + " -1234567890"
            for htmlline in html:
                if "citation_author" in htmlline:
                    auname = htmlline.split("citation_author\" content=\"")[1]
                    auname = auname.split(",")[0].lower()
                    auname = lxml.html.fromstring(auname).text_content()
                    auname = "".join([i for i in unidecode(unicode(auname.lower())) if i in valid_letters])
                    author_surnames.append(auname)
            author_list = "".join([author + " " for author in author_surnames])[:-1]
            print "authors: ", author_list
            for htmlline in html:
                if "citation_title" in htmlline:
                    title = htmlline.split("citation_title\" content=\"")[1]
                    title = title.split("\"")[0].lower()
                    title = lxml.html.fromstring(title).text_content()
                    title = "".join([i for i in unidecode(unicode(title.lower())) if i in valid_letters])[:75]
                    break
            arxidv_name = arxidv
            if "/" in arxidv_name:
                # This is some special-casing needed for old-style
                # arXiv IDs (such as math/0112073), since the slash
                # would confuse the file system.
                arxidv_name = arxidv_name.split("/")[1]
            resulting_filename = author_list + " - " + title + " - " + arxidv_name
            print "downloading as: ", resulting_filename
            # Downloading. The "while readsize" loop is meant to protect
            # against some temporary failures that haven't been occurring
            # lately.
            # Beware: It is stupid and might create an endless loop.
            readsize = 0
            while readsize == 0:
                urllib.urlretrieve(arxivprefix + "/pdf/" + arxidv, resulting_filename + ".pdf")
                readsize = os.stat(resulting_filename + ".pdf").st_size
                if readsize > 4500:
                    break
                testopen = open(resulting_filename + ".pdf")
                for line in testopen:
                    if "may take a little time" in line:
                        time.sleep(4)
                        readsize = 0
                        print "retrying..."
                        break
                else:
                    readsize = 6666
                testopen.close()
            urllib.urlretrieve(arxivprefix + "/e-print/" + arxidv, resulting_filename + ".tar.gz")
            print "\n resting..."
            time.sleep(resting_time)

if proper_mail:
    mail.close()

Answer 7

This is probably not quite the right fit for your use case, but it should be mentioned: Arxiv does have several relevant help pages:

• https://arxiv.org/help/robots
• https://arxiv.org/help/bulk_data
• https://arxiv.org/help/bulk_data_s3

The TLDR is that there are various interfaces for programmatic access to arxiv, but indiscriminate mass downloads will be blocked.

Answer 8

If you prefer a browser plugin to automatically rename downloaded PDFs while browsing, you may want to install arxiv-utils (I'm the author). The plugin simply adds a Direct Download button in arxiv web pages, and the naming convention is customizable.