# test/test_indent.py

#!/usr/bin/env python3

import sys
import lxml.etree as etree

from anthology.utils import indent

filename = "data/test.xml"
tree = etree.parse(filename)
root = tree.getroot()
indent(root)
print(etree.tostring(root).decode("utf-8"))


# tests/test_tex_math.py

import pytest
from lxml import etree
from anthology.texmath import TexMath

texmath = TexMath()

test_cases_unicode = (
    (
        '<tex-math>K</tex-math>-Embeddings: Learning Conceptual Embeddings for Words using Context',
        'K-Embeddings: Learning Conceptual Embeddings for Words using Context',
    ),
    (
        '<tex-math>^{\\mathcal{E}}</tex-math>: a Vectorial Resource for Computing Conceptual Similarity',
        'ℰ: a Vectorial Resource for Computing Conceptual Similarity',
    ),
    (
        '<tex-math>\\sharp</tex-math>: An Enhancement Approach to Reference-based Evaluation Metrics for Open-domain Dialog Generation',
        '♯: An Enhancement Approach to Reference-based Evaluation Metrics for Open-domain Dialog Generation',
    ),
    ('<tex-math>k</tex-math>-arization of Synchronous ', 'k-arization of Synchronous '),
    ('<tex-math>N</tex-math>-Gram Translation', 'N-Gram Translation'),
    ('<tex-math>k</tex-math>-best ', 'k-best '),
    (
        '<tex-math>n</tex-math>-grams – Investigating Abstraction and Domain Dependence',
        'n-grams – Investigating Abstraction and Domain Dependence',
    ),
    (
        '<tex-math>N</tex-math>-gram Fragment Sequence Based Unsupervised Domain-Specific Document Readability',
        'N-gram Fragment Sequence Based Unsupervised Domain-Specific Document Readability',
    ),
    ('<tex-math>n</tex-math>-grams', 'n-grams'),
    ('<tex-math>O(M(n^2))</tex-math> Time', 'O(M(n2)) Time'),
    ('<tex-math>L_0</tex-math>-norm', 'L0-norm'),
    ('<tex-math>n</tex-math>-gram embedding', 'n-gram embedding'),
    ('<tex-math>N</tex-math>-best List Re-ranking', 'N-best List Re-ranking'),
    ('<tex-math>O(n^6)</tex-math> ', 'O(n6) '),
    (
        '<tex-math>^{\\circ}</tex-math>: A Referring Expression Recognition Dataset in 360',
        '∘: A Referring Expression Recognition Dataset in 360',
    ),
    ('<tex-math>^{\\circ}</tex-math> Images', '∘ Images'),
    ('<tex-math>k</tex-math>', 'k'),
    (
        '<tex-math>k</tex-math>-Fold Ensemble for Out-Of-Distribution Detection',
        'k-Fold Ensemble for Out-Of-Distribution Detection',
    ),
    ('<tex-math>Q^{2}</tex-math>: ', 'Q2: '),
    ('<tex-math>l_{0}</tex-math>-norm-based Alignment', 'l0-norm-based Alignment'),
    ('<tex-math>\\tau</tex-math> Maximization', '𝜏 Maximization'),
    (
        '<tex-math>\\ell_1</tex-math>-Norm Symmetric Nonnegative Matrix Factorization',
        'ℓ1-Norm Symmetric Nonnegative Matrix Factorization',
    ),
    (
        '<tex-math>{\\mathcal{P}^2}</tex-math>: A Plan-and-Pretrain Approach for Knowledge Graph-to-Text Generation',
        '𝒫2: A Plan-and-Pretrain Approach for Knowledge Graph-to-Text Generation',
    ),
    (
        '<tex-math>^2</tex-math> Learning: Actively reducing redundancies in Active Learning methods for Sequence Tagging and Machine Translation',
        '2 Learning: Actively reducing redundancies in Active Learning methods for Sequence Tagging and Machine Translation',
    ),
    ('<tex-math>\\ell_{1}</tex-math> Norm Optimisation', 'ℓ1 Norm Optimisation'),
    (
        '<tex-math>S^3</tex-math> - Statistical Sandhi Splitting',
        'S3 - Statistical Sandhi Splitting',
    ),
    ('<tex-math>\\epsilon</tex-math>-extension Hidden ', '𝜖-extension Hidden '),
    (
        '<tex-math>\\varepsilon</tex-math>-Skip Discriminating-Reverse Parsing on Graph-Structured Stack',
        '𝜀-Skip Discriminating-Reverse Parsing on Graph-Structured Stack',
    ),
    (
        '<tex-math>F^2</tex-math> - New Technique for Recognition of User Emotional States in Spoken Dialogue Systems',
        'F2 - New Technique for Recognition of User Emotional States in Spoken Dialogue Systems',
    ),
    (
        '<tex-math>0(n^6)</tex-math> Recognition Algorithm for Mildly Context-Sensitive Languages',
        '0(n6) Recognition Algorithm for Mildly Context-Sensitive Languages',
    ),
    ('<tex-math>\\lambda</tex-math>-', '𝜆-'),
    ('<tex-math>\\leftrightarrow</tex-math> ', '↔ '),
    ('<tex-math>\\Phi</tex-math>', '𝛷'),
)

test_cases_html = (
    (
        '<tex-math>K</tex-math>-Embeddings: Learning Conceptual Embeddings for Words using Context',
        '<span class="tex-math">K</span>-Embeddings: Learning Conceptual Embeddings for Words using Context',
    ),
    (
        '<tex-math>^{\\mathcal{E}}</tex-math>: a Vectorial Resource for Computing Conceptual Similarity',
        '<span class="tex-math"><sup>ℰ</sup></span>: a Vectorial Resource for Computing Conceptual Similarity',
    ),
    (
        '<tex-math>\\sharp</tex-math>: An Enhancement Approach to Reference-based Evaluation Metrics for Open-domain Dialog Generation',
        '<span class="tex-math">♯</span>: An Enhancement Approach to Reference-based Evaluation Metrics for Open-domain Dialog Generation',
    ),
    (
        '<tex-math>k</tex-math>-arization of Synchronous ',
        '<span class="tex-math">k</span>-arization of Synchronous ',
    ),
    (
        '<tex-math>N</tex-math>-Gram Translation',
        '<span class="tex-math">N</span>-Gram Translation',
    ),
    ('<tex-math>k</tex-math>-best ', '<span class="tex-math">k</span>-best '),
    (
        '<tex-math>n</tex-math>-grams – Investigating Abstraction and Domain Dependence',
        '<span class="tex-math">n</span>-grams – Investigating Abstraction and Domain Dependence',
    ),
    (
        '<tex-math>N</tex-math>-gram Fragment Sequence Based Unsupervised Domain-Specific Document Readability',
        '<span class="tex-math">N</span>-gram Fragment Sequence Based Unsupervised Domain-Specific Document Readability',
    ),
    ('<tex-math>n</tex-math>-grams', '<span class="tex-math">n</span>-grams'),
    (
        '<tex-math>O(M(n^2))</tex-math> Time',
        '<span class="tex-math">O(M(n<sup>2</sup>))</span> Time',
    ),
    ('<tex-math>L_0</tex-math>-norm', '<span class="tex-math">L<sub>0</sub></span>-norm'),
    (
        '<tex-math>n</tex-math>-gram embedding',
        '<span class="tex-math">n</span>-gram embedding',
    ),
    (
        '<tex-math>N</tex-math>-best List Re-ranking',
        '<span class="tex-math">N</span>-best List Re-ranking',
    ),
    ('<tex-math>O(n^6)</tex-math> ', '<span class="tex-math">O(n<sup>6</sup>)</span> '),
    (
        '<tex-math>^{\\circ}</tex-math>: A Referring Expression Recognition Dataset in 360',
        '<span class="tex-math"><sup>∘</sup></span>: A Referring Expression Recognition Dataset in 360',
    ),
    (
        '<tex-math>^{\\circ}</tex-math> Images',
        '<span class="tex-math"><sup>∘</sup></span> Images',
    ),
    ('<tex-math>k</tex-math>', '<span class="tex-math">k</span>'),
    (
        '<tex-math>k</tex-math>-Fold Ensemble for Out-Of-Distribution Detection',
        '<span class="tex-math">k</span>-Fold Ensemble for Out-Of-Distribution Detection',
    ),
    ('<tex-math>Q^{2}</tex-math>: ', '<span class="tex-math">Q<sup>2</sup></span>: '),
    (
        '<tex-math>l_{0}</tex-math>-norm-based Alignment',
        '<span class="tex-math">l<sub>0</sub></span>-norm-based Alignment',
    ),
    (
        '<tex-math>\\tau</tex-math> Maximization',
        '<span class="tex-math">𝜏</span> Maximization',
    ),
    (
        '<tex-math>\\ell_1</tex-math>-Norm Symmetric Nonnegative Matrix Factorization',
        '<span class="tex-math">ℓ<sub>1</sub></span>-Norm Symmetric Nonnegative Matrix Factorization',
    ),
    (
        '<tex-math>{\\mathcal{P}^2}</tex-math>: A Plan-and-Pretrain Approach for Knowledge Graph-to-Text Generation',
        '<span class="tex-math">𝒫<sup>2</sup></span>: A Plan-and-Pretrain Approach for Knowledge Graph-to-Text Generation',
    ),
    (
        '<tex-math>^2</tex-math> Learning: Actively reducing redundancies in Active Learning methods for Sequence Tagging and Machine Translation',
        '<span class="tex-math"><sup>2</sup></span> Learning: Actively reducing redundancies in Active Learning methods for Sequence Tagging and Machine Translation',
    ),
    (
        '<tex-math>\\ell_{1}</tex-math> Norm Optimisation',
        '<span class="tex-math">ℓ<sub>1</sub></span> Norm Optimisation',
    ),
    (
        '<tex-math>S^3</tex-math> - Statistical Sandhi Splitting',
        '<span class="tex-math">S<sup>3</sup></span> - Statistical Sandhi Splitting',
    ),
    (
        '<tex-math>\\epsilon</tex-math>-extension Hidden ',
        '<span class="tex-math">𝜖</span>-extension Hidden ',
    ),
    (
        '<tex-math>\\varepsilon</tex-math>-Skip Discriminating-Reverse Parsing on Graph-Structured Stack',
        '<span class="tex-math">𝜀</span>-Skip Discriminating-Reverse Parsing on Graph-Structured Stack',
    ),
    (
        '<tex-math>F^2</tex-math> - New Technique for Recognition of User Emotional States in Spoken Dialogue Systems',
        '<span class="tex-math">F<sup>2</sup></span> - New Technique for Recognition of User Emotional States in Spoken Dialogue Systems',
    ),
    (
        '<tex-math>0(n^6)</tex-math> Recognition Algorithm for Mildly Context-Sensitive Languages',
        '<span class="tex-math">0(n<sup>6</sup>)</span> Recognition Algorithm for Mildly Context-Sensitive Languages',
    ),
    ('<tex-math>\\lambda</tex-math>-', '<span class="tex-math">𝜆</span>-'),
    ('<tex-math>\\leftrightarrow</tex-math> ', '<span class="tex-math">↔</span> '),
    ('<tex-math>\\Phi</tex-math>', '<span class="tex-math">𝛷</span>'),
    # Manually collected tests
    (
        '<tex-math>0(n^{\\tilde{\\rho}+1})</tex-math>',
        '<span class="tex-math">0(n<sup> ̃𝜌+1</sup>)</span>',
    ),
    (
        '<tex-math>\\{mt, src\\} \\rightarrow pe</tex-math>',
        '<span class="tex-math">{mt, src} → pe</span>',
    ),
    (
        '<tex-math>p(\\boldsymbol{y}|\\textrm{do}(\\boldsymbol{x}))</tex-math>',
        '<span class="tex-math">p(<strong>y</strong>|<span>do</span>(<strong>x</strong>))</span>',
    ),
    ('<tex-math>{\\sim}3\\%</tex-math>', '<span class="tex-math">∼3%</span>'),
    (
        '<tex-math>O(\\log_2 n)</tex-math>',
        '<span class="tex-math">O(<span class="tex-math-function">log</span><sub>2</sub> n)</span>',
    ),
    (
        '<tex-math>\\mathbf{^2}</tex-math>',
        '<span class="tex-math"><strong><sup>2</sup></strong></span>',
    ),
    (
        '<tex-math>RoBERTa_{large}</tex-math>',
        '<span class="tex-math">RoBERTa<sub>large</sub></span>',
    ),
    ('<tex-math>\\ell_1</tex-math>', '<span class="tex-math">ℓ<sub>1</sub></span>'),
    (
        '<tex-math>n \\log_2 \\frac{m}{n} + o(m)</tex-math>',
        '<span class="tex-math">n <span class="tex-math-function">log</span><sub>2</sub> <sup>m</sup>⁄<sub>n</sub> + o(m)</span>',
    ),
    (
        '<tex-math>\textrm{Pr}(f_1^J/e^I_1)</tex-math>',
        '<span class="tex-math">\textrmPr(f<sub>1</sub><sup>J</sup>/e<sup>I</sup><sub>1</sub>)</span>',
    ),
    (
        '<tex-math>\\# \\$ \\% \\&amp; \\_ \\{ \\} \\| \\:</tex-math>',
        '<span class="tex-math"># $ % &amp; _ { } ‖  </span>',
    ),
    (
        '<tex-math>asd\\$asd</tex-math>',
        '<span class="tex-math">asd$asd</span>',
    ),
    (
        '<tex-math>2\\_3</tex-math>',
        '<span class="tex-math">2_3</span>',
    ),
    (
        '<tex-math>2_3</tex-math>',
        '<span class="tex-math">2<sub>3</sub></span>',
    ),
)


@pytest.mark.parametrize('inp, out', test_cases_unicode)
def test_unicode(inp, out):
    element = etree.fromstring(f"<span>{inp}</span>")
    math_element = element.find(".//tex-math")
    actual_out = texmath.to_unicode(math_element)
    assert actual_out == out


@pytest.mark.parametrize('inp, out', test_cases_html)
def test_html(inp, out):
    element = etree.fromstring(f"<span>{inp}</span>")
    math_element = element.find(".//tex-math")
    result = texmath.to_html(math_element)
    actual_out = etree.tostring(result, encoding="unicode")
    assert actual_out == out


# bin/anthology/__init__.py

from .anthology import Anthology
from .people import PersonName
from .papers import Paper
from .volumes import Volume


# bin/anthology/anthology.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from glob import glob
from lxml import etree
import logging as log
import os

from datetime import datetime
import pytz

from .formatter import MarkupFormatter
from .index import AnthologyIndex
from .papers import Paper
from .venues import VenueIndex
from .volumes import Volume
from .sigs import SIGIndex


class Anthology:
    schema = None
    pindex = None
    venues = None
    sigs = None
    formatter = None

    def __init__(self, importdir=None, fast_load=False, require_bibkeys=True):
        """Instantiates the Anthology.

        :param importdir: Data directory to import from; if not given, you'll
        need to call `import_directory` explicitly to load the Anthology data.
        :param fast_load: If True, will disable some error checking in favor
        of faster loading.
        :param require_bibkeys: If True (default), will log errors if papers
        don't have a bibkey; can be set to False in order to create bibkeys
        for newly added papers.
        """
        self.formatter = MarkupFormatter()
        self.volumes = {}  # maps volume IDs to Volume objects
        self.papers = {}  # maps paper IDs to Paper objects
        self._fast_load = fast_load
        self._require_bibkeys = require_bibkeys
        if importdir is not None:
            self.import_directory(importdir)

    @property
    def people(self):
        # compatibility, since this was renamed internally
        return self.pindex

    def import_directory(self, importdir):
        assert os.path.isdir(importdir), f"Directory not found: {importdir}"
        self.pindex = AnthologyIndex(
            importdir,
            fast_load=self._fast_load,
            require_bibkeys=self._require_bibkeys,
            parent=self,
        )
        self.venues = VenueIndex(importdir)
        self.sigs = SIGIndex(importdir)
        for xmlfile in glob(importdir + "/xml/*.xml"):
            self.import_file(xmlfile)
        self.pindex.verify()

    def import_file(self, filename):
        # furthest east!
        # date_in_kiritimati = datetime.now(pytz.timezone("Pacific/Kiritimati")).date()

        tree = etree.parse(filename)
        collection = tree.getroot()
        collection_id = collection.get("id")
        for volume_xml in collection:
            volume = Volume.from_xml(
                volume_xml,
                collection_id,
                self.venues,
                self.sigs,
                self.formatter,
            )

            # MJP 2021-05: no longer doing this since it kills branch previews.
            # Don't merge with master prior to ingest date!
            #
            # skip volumes that have an ingestion date in the future
            # if (
            #     datetime.strptime(volume.ingest_date, "%Y-%m-%d").date()
            #     > date_in_kiritimati
            # ):
            #     log.info(
            #         f"Skipping volume {volume.full_id} with ingestion date {volume.ingest_date} in the future."
            #     )
            #     # Remove any SIG entries with this volume
            #     self.sigs.remove_volume(volume.full_id)
            #     continue

            # Register the volume since we're not skipping it
            self.venues.register(volume)

            if volume.full_id in self.volumes:
                log.critical(f"Attempted to import volume ID '{volume.full_id}' twice")
                log.critical(f"Triggered by file: {filename}")

            # front matter
            if volume.has_frontmatter:
                front_matter = volume.content[0]
                self.pindex.register(front_matter)
                self.papers[front_matter.full_id] = front_matter
            else:
                # dummy front matter to make sure that editors of
                # volume get registered as people in author database
                dummy_front_matter = Paper("0", None, volume, self.formatter)
                self.pindex.register(dummy_front_matter, dummy=True)

            self.volumes[volume.full_id] = volume
            for paper_xml in volume_xml.findall("paper"):
                parsed_paper = Paper.from_xml(
                    paper_xml, volume, self.formatter, self.venues
                )

                # MJP 2021-05: no longer doing this since it kills branch previews.
                # Don't merge with master prior to ingest date!
                #
                # skip papers that have an ingestion date in the future
                # if (
                #     datetime.strptime(parsed_paper.ingest_date, "%Y-%m-%d").date()
                #     > date_in_kiritimati
                # ):
                #     log.info(
                #         f"Skipping paper {parsed_paper.full_id} with ingestion date {parsed_paper.ingest_date} in the future."
                #     )
                #     continue

                self.pindex.register(parsed_paper)
                full_id = parsed_paper.full_id
                if full_id in self.papers:
                    log.critical(
                        f"Attempted to import paper '{full_id}' twice -- skipping"
                    )
                    continue
                volume.append(parsed_paper)
                self.papers[full_id] = parsed_paper


# bin/anthology/bibtex.py

# -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import re
import logging
import pybtex, pybtex.database.input.bibtex


def read_bibtex(bibfilename):
    """
    Reads bibtex data from a file into pybtex data structures.
    """

    bibbytes = open(bibfilename, "rb").read()
    bibstring = None
    for encoding in ['ascii', 'utf8', 'cp1252']:
        try:
            bibstring = bibbytes.decode(encoding)
        except UnicodeDecodeError:
            continue
        logging.debug(f"{bibfilename}: using {encoding} encoding")
        break
    else:
        logging.warning("couldn't figure out encoding; using ascii with escapes")
        bibstring = bibbytes.decode('ascii', 'backslashreplace')

    if bibstring.startswith('\uFEFF'):
        bibstring = bibstring[1:]  # Unicode BOM

    # for parser in [lambda s: pybtex.database.parse_string(s, 'bibtex'),
    #                fake_parse]:
    try:
        bibdata = pybtex.database.parse_string(bibstring, "bibtex")
    except Exception as e:
        import sys

        print(f"Failed to parse {bibfilename}", file=sys.stderr)
        print(e)
        sys.exit(1)
    return bibdata


# bin/anthology/data.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

################################################################################
# This file contains all constants and functions that have hardcoded data (such
# as URLs or journal titles) which does not come from the XML.  This is to
# provide a single file where such hardcoded data can be looked up and/or
# changed.
################################################################################

import os

# this is the canonical URL.  In contrast to all other
# URL templates, it always links to the official anthology.
CANONICAL_URL_TEMPLATE = "https://aclanthology.org/{}"

# the prefix is used in different programs and we need to set it everywhere
# We use a environment variable to set this and not have to forward the value
# through all the programs. If this does not look like the best idea, keep in mind
# that the structure is historically grown -- from 2019 to 2020 :-)
try:
    ANTHOLOGY_PREFIX = os.environ["ANTHOLOGY_PREFIX"]
except:
    ANTHOLOGY_PREFIX = "https://aclanthology.org"

ATTACHMENT_PREFIX = ANTHOLOGY_PREFIX + "/attachments"
ATTACHMENT_TEMPLATE = ATTACHMENT_PREFIX + "/{}"

PDF_LOCATION_TEMPLATE = ANTHOLOGY_PREFIX + "/{}.pdf"
PDF_THUMBNAIL_LOCATION_TEMPLATE = ANTHOLOGY_PREFIX + "/thumb/{}.jpg"

VIDEO_LOCATION_TEMPLATE = ANTHOLOGY_PREFIX + "/{}"

# Regular expression matching full Anthology IDs
ANTHOLOGY_ID_REGEX = r"[A-Z]\d{2}-\d{4}"

# Anthology file location on server
# Defaults to ~/anthology-files
ANTHOLOGY_FILE_DIR = os.environ.get(
    "ANTHOLOGY_FILES", os.path.join(os.environ["HOME"], "anthology-files")
)

# Names of XML elements that may appear multiple times
LIST_ELEMENTS = (
    "attachment",
    "author",
    "editor",
    "video",
    "revision",
    "erratum",
    "award",
    "pwcdataset",
    "video",
)

# New-style IDs that should be handled as journals
JOURNAL_IDS = ("cl", "tacl")

# Constants associated with DOI assignation
DOI_URL_PREFIX = "https://dx.doi.org/"
DOI_PREFIX = "10.18653/v1/"

# Default ingestion date (= unknown)
UNKNOWN_INGEST_DATE = "1900-01-01"

# The venue format must match this pattern
VENUE_FORMAT = r"^[A-Za-z\d]+$"


def get_journal_title(top_level_id, volume_title):
    # TODO: consider moving this from code to data (perhaps
    # under <booktitle> in the volume metadata

    top_level_id = top_level_id.split(".")[-1]  # for new-style IDs; is a no-op otherwise

    if top_level_id == "cl":
        return "Computational Linguistics"
    if top_level_id[0] == "J":
        year = int(top_level_id[1:3])
        if year >= 65 and year <= 83:
            return "American Journal of Computational Linguistics"
        else:
            return "Computational Linguistics"
    elif top_level_id[0] == "Q" or top_level_id == "tacl":
        return "Transactions of the Association for Computational Linguistics"
    else:
        return volume_title


# bin/anthology/formatter.py

# -*- coding: utf-8 -*-
#
# Copyright 2019-2022 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging as log
from copy import deepcopy
from lxml import etree
import citeproc
from citeproc.source.json import CiteProcJSON
import citeproc_styles
import codecs
import re

from . import latexcodec
from .texmath import TexMath
from .utils import stringify_children, remove_extra_whitespace


latexcodec.register()


_BIBTEX_MONTHS = {
    "january": "jan",
    "february": "feb",
    "march": "mar",
    "april": "apr",
    "may": "may",
    "june": "jun",
    "july": "jul",
    "august": "aug",
    "september": "sep",
    "october": "oct",
    "november": "nov",
    "december": "dec",
}


class CiteprocFormatter:
    """Formatter using Citeproc and CSL files to produce citations.

    cf. https://github.com/citation-style-language/styles for possible citation
    styles
    """

    styles = {}

    @classmethod
    def load_style(cls, style):
        """Loads and returns a CSL style."""
        if style not in cls.styles:
            cls.styles[style] = citeproc.CitationStylesStyle(
                citeproc_styles.get_style_filepath(style)
            )
        return cls.styles[style]

    @classmethod
    def render_html_citation(
        cls, paper, style="association-for-computational-linguistics"
    ):
        """Render a bibliography entry for a paper with a given CSL style.

        Returns HTML encoded as a single string.
        """
        data = paper.as_citeproc_json()
        source = CiteProcJSON(data)
        item = citeproc.CitationItem(data[0]["id"])
        bib = citeproc.CitationStylesBibliography(
            cls.load_style(style), source, citeproc.formatter.html
        )
        bib.register(citeproc.Citation([item]))
        return str(bib.style.render_bibliography([item])[0])


def bibtex_encode(text):
    """Encodes a text string for use in BibTeX.

    Assumes that the text does *not* contain any LaTeX commands!
    """
    if text is None:
        return ""
    text = codecs.encode(text, "latex")
    return text


def bibtex_convert_quotes(text):
    if re.match(r"(?<!\\)\"", text):
        log.warning(
            f'Straight quote (") found in text field ({text}); converting automatically, but please fix in XML'
        )
    text = re.sub(r"(?<!\\)\"\b", "``", text)
    text = re.sub(r"(?<!\\)\"", "''", text)
    return text


def bibtex_convert_month(text):
    """Converts a month string to BibTeX macros.

    If the string contains digits or is otherwise not parseable, it is returned
    unchanged with quotes around it.
    """
    text = text.lower()
    if text in _BIBTEX_MONTHS:  # most common case; map e.g. march -> mar
        return _BIBTEX_MONTHS[text]
    if text in _BIBTEX_MONTHS.values():  # already a month spec
        return text
    # Find embedded month strings
    text = f'"{text}"'
    for month, macro in _BIBTEX_MONTHS.items():
        if month in text:
            text = text.replace(month, f'" # {macro} # "')
            text = " # ".join(filter(lambda k: k != '""', text.split(" # ")))
    return text


def bibtex_make_entry(bibkey, bibtype, fields):
    lines = [f"@{bibtype}{{{bibkey},"]
    for key, value in fields:
        if key == "author" and bibtype == "proceedings":
            key = "editor"
        if key in ("author", "editor") and "  and  " in value:
            # Print each author on a separate line
            value = "  and\n      ".join(value.split("  and  "))
        if key == "month":
            value = bibtex_convert_month(value)
        elif value is None:
            log.warning(f"Skipping empty value for {bibkey}/{key}")
            continue
        elif '"' in value:
            # Make sure not to use "" to quote values when they contain "
            value = f"{{{value}}}"
        else:
            # quote value
            value = f'"{value}"'
        lines.append(f"    {key} = {value},")
    lines.append("}")
    return "\n".join(lines)


class MarkupFormatter:
    def __init__(self):
        self.texmath = TexMath()

    def as_xml(self, element):
        return remove_extra_whitespace(stringify_children(element))

    def as_text(self, element):
        element = deepcopy(element)
        for sub in element.iterfind(".//tex-math"):
            sub.text = self.texmath.to_unicode(sub)
        retval = etree.tostring(element, encoding="unicode", method="text")
        return remove_extra_whitespace(retval)

    def as_html(self, element, allow_url=False):
        element = deepcopy(element)
        # Transform elements to valid HTML
        for sub in element.iterfind(".//url"):
            if allow_url:
                sub.tag = "a"
                sub.attrib["href"] = sub.text
            else:
                sub.tag = "span"
            sub.attrib["class"] = "acl-markup-url"
        for sub in element.iterfind(".//fixed-case"):
            sub.tag = "span"
            sub.attrib["class"] = "acl-fixed-case"
        for sub in element.iterfind(".//tex-math"):
            parsed_elem = self.texmath.to_html(sub)
            parsed_elem.tail = sub.tail
            sub.getparent().replace(sub, parsed_elem)
        retval = stringify_children(element)
        return remove_extra_whitespace(retval)

    def as_latex(self, element):
        # following convert_xml_text_markup in anth2bib.py
        if element.tag in ["tex-math", "url"]:
            if len(element) > 0:
                log.warning(f"<{element.tag}> element has children")
            text = element.text
        else:
            text = bibtex_encode(element.text)
        for nested_element in element:
            text += self.as_latex(nested_element)
            text += bibtex_encode(nested_element.tail)
        if element.tag == "fixed-case":
            text = f"{{{text}}}"
        elif element.tag == "b":
            text = f"\\textbf{{{text}}}"
        elif element.tag == "i":
            text = f"\\textit{{{text}}}"
        elif element.tag == "tex-math":
            text = f"${text}$"
        elif element.tag == "url":
            text = f"\\url{{{text}}}"
        text = bibtex_convert_quotes(text)
        return remove_extra_whitespace(text)

    def __call__(self, element, form, **kwargs):
        if element is None:
            return ""
        if form == "xml":
            return self.as_xml(element)
        elif form in ("plain", "text"):
            return self.as_text(element)
        elif form == "html":
            return self.as_html(element, **kwargs)
        elif form == "latex":
            return self.as_latex(element)
        raise ValueError(f"Unknown format: {form}")


# bin/anthology/index.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging as log
import re
import yaml
from collections import defaultdict, Counter
from functools import lru_cache
import itertools as it
from slugify import slugify
from stop_words import get_stop_words
from .formatter import bibtex_encode
from .people import PersonName
from .venues import VenueIndex

from typing import List

try:
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader


BIBKEY_MAX_NAMES = 2


def load_stopwords(language):
    return [t for w in get_stop_words(language) for t in slugify(w).split("-")]


# Temporary hack until we refactor person/name handling
class defaultdict_names(defaultdict):
    """This is a defaultdict that indexes PersonName objects, but without regard
    for any locally defined name variant, so that PersonName('X', 'Y') and
    PersonName('X', 'Y', variant=foo) will key to the same thing.
    """

    def __getitem__(self, key: PersonName):
        return super().__getitem__(key.without_variant())

    def __setitem__(self, key: PersonName, val):
        return super().__setitem__(key.without_variant(), val)

    def __delitem__(self, key: PersonName):
        return super().__delitem__(key.without_variant())

    def __contains__(self, key: PersonName):
        return super().__contains__(key.without_variant())

    def get(self, key: PersonName, default=None):
        return super().get(key.without_variant(), default)


class AnthologyIndex:
    """Keeps an index of people and papers.

    This class provides:
    - An index of people (authors/editors) with their internal IDs, canonical
      names, and name variants.
    - A mapping of people to all papers associated with them.
    - A set of all bibliography keys used within the Anthology and a method to
      create new ones, guaranteeing uniqueness.

    The index is NOT automatically populated when instantiating this class, but
    rather gets its data from papers being registered in it as they are loaded
    from the XML by the main `Anthology` class.

    :param srcdir: Path to the Anthology data directory. Only used for loading
    the list of name variants.
    :param fast_load: Whether to disable some error checking in favor of faster
    loading.
    :param require_bibkeys: Whether to log an error when a paper being added
    does not have a bibkey. Should only be set to False during the ingestion of
    new papers, when this class is being used to generate new, unique bibkeys.
    """

    def __init__(self, srcdir=None, fast_load=False, require_bibkeys=True, parent=None):
        self._parent = parent
        self._fast_load = fast_load
        self._require_bibkeys = require_bibkeys
        self.bibkeys = set()
        self.stopwords = load_stopwords("en")
        self.id_to_canonical = {}  # maps ids to canonical names
        self._id_to_used = defaultdict(set)  # maps ids to all names actually used
        self.name_to_ids = defaultdict_names(list)  # maps canonical/variant names to ids
        self._coauthors = defaultdict(Counter)  # maps ids to co-author ids
        self.comments = (
            {}
        )  # maps ids to comments (used for distinguishing authors with same name)
        self._similar = defaultdict(set)
        self.id_to_papers = defaultdict(lambda: defaultdict(list))  # id -> role -> papers
        self.name_to_papers = defaultdict_names(
            lambda: defaultdict(list)
        )  # name -> (explicit id?) -> papers; used only for error checking
        if srcdir is not None:
            self.load_variant_list(srcdir)

    def load_variant_list(self, directory):
        with open(f"{directory}/yaml/name_variants.yaml", "r") as f:
            name_list = yaml.load(f, Loader=Loader)

            # Reserve ids for people with explicit ids in variant list
            for entry in name_list:
                if "id" in entry:
                    id_ = entry["id"]
                    canonical = entry["canonical"]
                    canonical = PersonName.from_dict(canonical)
                    self.set_canonical_name(id_, canonical)
            # Automatically add people with same canonical name to similar list
            if not self._fast_load:
                for name, ids in self.name_to_ids.items():
                    if len(ids) > 1:
                        for (id1, id2) in it.permutations(ids, 2):
                            self._similar[id1].add(id2)
            for entry in name_list:
                try:
                    canonical = entry["canonical"]
                    variants = entry.get("variants", [])
                    id_ = entry.get("id", None)
                except (KeyError, TypeError):
                    log.error(f"Couldn't parse name variant entry: {entry}")
                    continue
                canonical = PersonName.from_dict(canonical)
                if id_ is None:
                    if canonical in self.name_to_ids:
                        log.error(
                            f"Canonical name '{canonical}' is ambiguous but doesn't have an id; please add one"
                        )
                    id_ = canonical.slug
                    self.set_canonical_name(id_, canonical)
                for variant in variants:
                    variant = PersonName.from_dict(variant)
                    if variant in self.name_to_ids:
                        log.error(
                            "Tried to add '{}' as variant of '{}', but is already a variant of '{}'".format(
                                repr(variant),
                                repr(canonical),
                                repr(self.id_to_canonical[self.name_to_ids[variant][0]]),
                            )
                        )
                        continue
                    self.add_variant_name(id_, variant)
                if "comment" in entry:
                    self.comments[id_] = entry["comment"]
                if "similar" in entry and not self._fast_load:
                    self._similar[id_].update(entry["similar"])
                    for other in entry["similar"]:
                        if id_ not in self._similar[other]:
                            log.debug(f'inferring similar name {other} -> {id_}')
                        self._similar[other].add(id_)

        # form transitive closure of self._similar
        if not self._fast_load:
            again = True
            while again:
                again = False
                for x in list(self._similar):
                    for y in list(self._similar[x]):
                        for z in list(self._similar[y]):
                            if z != x and z not in self._similar[x]:
                                self._similar[x].add(z)
                                log.debug(f'inferring similar name {x} -> {z}')
                                again = True

    def _is_stopword(self, word, paper):
        """Determines if a given word should be considered a stopword for
        the purpose of generating BibTeX keys."""
        if word in self.stopwords:
            return True
        if paper.is_volume:
            # Some simple heuristics to exclude probably uninformative words
            # -- these are not perfect
            if word in (
                "proceedings",
                "volume",
                "conference",
                "workshop",
                "annual",
                "meeting",
                "computational",
            ):
                return True
            elif (
                re.match(r"[0-9]+(st|nd|rd|th)", word)
                or word.endswith("ieth")
                or word.endswith("enth")
                or word
                in (
                    "first",
                    "second",
                    "third",
                    "fourth",
                    "fifth",
                    "sixth",
                    "eighth",
                    "ninth",
                    "twelfth",
                )
            ):
                return True
        return False

    def create_bibkey(self, paper, vidx=None):
        """Create a unique bibliography key for the given paper."""
        if self._fast_load:
            raise Exception(
                "Cannot create bibkeys when AnthologyIndex is instantiated with fast_load=True"
            )
        if paper.is_volume:
            # Proceedings volumes use venue acronym instead of authors/editors
            bibnames = slugify(vidx.get_main_venue(paper.full_id))
        else:
            # Regular papers use author/editor names
            names = paper.get("author")
            if not names:
                names = paper.get("editor", [])
            if names:
                if len(names) > BIBKEY_MAX_NAMES:
                    bibnames = f"{slugify(names[0][0].last)}-etal"
                else:
                    bibnames = "-".join(slugify(n.last) for n, _ in names)
            else:
                bibnames = "nn"
        title = [
            w
            for w in slugify(paper.get_title("plain")).split("-")
            if not self._is_stopword(w, paper)
        ]
        bibkey = f"{bibnames}-{paper.get('year')}-{title.pop(0)}"
        while bibkey in self.bibkeys:  # guarantee uniqueness
            if title:
                bibkey += f"-{title.pop(0)}"
            else:
                match = re.search(r"-([0-9][0-9]?)$", bibkey)
                if match is not None:
                    num = int(match.group(1)) + 1
                    bibkey = bibkey[: -len(match.group(1))] + f"{num}"
                else:
                    bibkey += "-2"
                log.debug(
                    f"New bibkey for clash that can't be resolved by adding title words: {bibkey}"
                )
        paper.bibkey = bibkey
        self.register_bibkey(paper)
        return bibkey

    def register_bibkey(self, paper):
        """Register a paper's bibkey in Anthology-wide set to ensure uniqueness."""
        key = paper.bibkey
        if key is None:
            if self._require_bibkeys:
                log.error(f"Paper {paper.full_id} has no bibkey!")
            return
        if key in self.bibkeys:
            log.error(f"Paper {paper.full_id} has bibkey that is not unique ({key})!")
            return
        self.bibkeys.add(key)

    def register(self, paper, dummy=False):
        """Register bibkey and names associated with the given paper.

        :param dummy: If True, will only resolve the author/editor names without
        actually linking them to the given paper.  This is used for volumes
        without frontmatter to make sure their editors still get registered
        here, but without creating links to a non-existent paper.
        """
        from .papers import Paper

        assert isinstance(
            paper, Paper
        ), f"Expected Paper, got {type(paper)} ({repr(paper)})"
        # Make sure paper has a bibkey and it is unique (except for dummy
        # frontmatter, as it is not an actual paper)
        if not dummy and not self._fast_load:
            self.register_bibkey(paper)
        # Resolve and register authors/editors for this paper
        for role in ("author", "editor"):
            for name, id_ in paper.get(role, []):
                if id_ is None:
                    if len(self.name_to_ids.get(name, [])) > 1:
                        log.error(
                            f"Paper {paper.full_id} uses ambiguous name '{name}' without id"
                        )
                        log.error(
                            "  Please add an id, for example: {}".format(
                                " ".join(self.name_to_ids[name])
                            )
                        )
                    id_ = self.resolve_name(name)["id"]
                    explicit = False
                else:
                    if id_ not in self.id_to_canonical:
                        log.error(
                            f"Paper {paper.full_id} uses name '{name}' with id '{id_}' that does not exist"
                        )
                    explicit = True

                if not self._fast_load:
                    self._id_to_used[id_].add(name)

                if not dummy:
                    # Register paper
                    self.id_to_papers[id_][role].append(paper.full_id)
                    if not self._fast_load:
                        self.name_to_papers[name][explicit].append(paper.full_id)
                        # Register co-author(s)
                        for co_name, co_id in paper.get(role):
                            if co_id is None:
                                co_id = self.resolve_name(co_name)["id"]
                            if co_id != id_:
                                self._coauthors[id_][co_id] += 1

    @property
    def id_to_used(self):
        if self._fast_load and not self._id_to_used:
            for paper in self._parent.papers.values():
                for (name, id_, _) in paper.iter_people():
                    self._id_to_used[id_].add(name)
        return self._id_to_used

    @property
    def coauthors(self):
        if self._fast_load and not self._coauthors:
            for paper in self._parent.papers.values():
                people = list(paper.iter_people())
                for (p1, p2) in it.permutations(people, 2):
                    name1, id1, role1 = p1
                    name2, id2, role2 = p2
                    if role1 != role2:
                        continue
                    if id1 is None:
                        id1 = self.resolve_name(name1)["id"]
                    if id2 is None:
                        id2 = self.resolve_name(name2)["id"]
                    self._coauthors[id1][id2] += 1
        return self._coauthors

    @property
    def similar(self):
        if self._fast_load:
            raise Exception(
                "Cannot retrieve list of similar names when AnthologyIndex is instantiated with fast_load=True"
            )
        return self._similar

    def verify(self):
        ## no longer issuing a warning for unused variants
        ## it is generally a good idea to keep them around in case they pop up again
        ## if you want to prune them, try clean_name_variants.py
        # for name, ids in self.name_to_ids.items():
        #    for id_ in ids:
        #        cname = self.id_to_canonical[id_]
        #        if name != cname and name not in self._id_to_used[id_]:
        #            log.warning(
        #                "Variant name '{}' of '{}' is not used".format(
        #                    repr(name), repr(cname)
        #                )
        #            )
        if self._fast_load:
            return  # does nothing
        for name, d in self.name_to_papers.items():
            # name appears with more than one explicit id and also
            # appears without id at least once
            if len(d[False]) > 0 and len(d[True]) > 1:
                log.error(f"Name '{name}' is ambiguous and is used without explicit id")
                log.error(
                    "  Please add an id to paper(s):   {}".format(" ".join(d[False]))
                )

    def personids(self):
        return self.id_to_canonical.keys()

    def get_canonical_name(self, id_):
        return self.id_to_canonical[id_]

    def set_canonical_name(self, id_, name):
        if (not id_ in self.id_to_canonical) or (
            name.score > self.id_to_canonical[id_].score
        ):
            # if name not seen yet, or if this version has more accents
            self.id_to_canonical[id_] = name
        self.name_to_ids[name].append(id_)

    def add_variant_name(self, id_, name):
        self.name_to_ids[name].append(id_)

    def get_used_names(self, id_):
        """Return a list of all names used for a given person."""
        return self.id_to_used[id_]

    def get_ids(self, name: PersonName) -> List[str]:
        """
        Returns a list of distinct IDs (people) associated with a surface form.

        :param name: The name (surface form) of the person being searched (id field ignored).
        :return: A list of name ID strings.
        """
        if name not in self.name_to_ids:
            id_ = name.slug
            self.set_canonical_name(id_, name)

        return sorted(self.name_to_ids[name])

    def get_comment(self, id_: str) -> str:
        """
        Returns the comment associated with the name ID.

        :param id_: The name ID (e.g., "fei-liu-utdallas")
        :return: The comment (e.g., "UT Dallas, Bosch, CMU, University of Central Florida")
        """
        return self.comments.get(id_, None)

    @lru_cache(maxsize=None)
    def resolve_name(self, name, id_=None):
        """Find person named 'name' and return a dict with fields
        'first', 'last', 'id'"""
        if id_ is None:
            ids = self.get_ids(name)
            assert len(ids) > 0
            if len(ids) > 1:
                log.debug(
                    "Name '{}' is ambiguous between {}".format(
                        repr(name), ", ".join(f"'{i}'" for i in ids)
                    )
                )
            # Just return the first
            id_ = ids[0]
        d = name.as_dict()
        d["id"] = id_
        return d

    def get_papers(self, id_, role=None):
        if role is None:
            return [p for p_list in self.id_to_papers[id_].values() for p in p_list]
        return self.id_to_papers[id_][role]

    def get_coauthors(self, id_):
        return self.coauthors[id_].items()

    def get_venues(self, vidx: VenueIndex, id_):
        """Get a list of venues a person has published in, with counts."""
        venues = Counter()
        for paper in self.get_papers(id_):
            for venue in vidx.get_associated_venues(paper):
                venues[venue] += 1
        return venues


# bin/anthology/latexcodec.py

# -*- coding: utf-8 -*-
#
# Copyright 2003 D. Eppstein
# Copyright 2019 Daniel Gildea
# Copyright 2019 David Wei Chiang
#
# The software is licensed according to the terms of the
# PSF (Python Software Foundation) license found here:
# http://www.python.org/psf/license/

"""Translates unicode to bibtex-friendly encoding.

bibtex-friendly features:

1) outputs "{\v{s}}" rather than "\v s" because bibtex
turns space into ~

2) wraps special chars in braces, for example "{\l}",
to prevent "\l" from eating the following space after
names are re-arranged by bibtex.

3) outputs the following commands recognized by bibtex's
purify$ command for alphabetization:
\i, \j, \oe, \OE, \ae, \AE, \aa, \AA, \o, \O, \l, \L, \ss

4) also outputs the following commands *not* recognized by purify$
 \th \TH \dh \DH \dj \DJ
in the interest of getting the char to show up in output pdf,
even though alphabetization will be wrong:
 "{\DJ}inh" will be alphabetized under "inh"

Dan Gildea 2019

Bugs:

- Doesn't convert "{" to "{\\{}" and "}" to "{\\}}" (which do
  occasionally appear in titles)

adapted from latex.py by D. Eppstein:

Character translation utilities for LaTeX-formatted text.

Usage:
 - unicode(string,'latex')
 - ustring.decode('latex')
are both available just by letting "import latex" find this file.
 - unicode(string,'latex+latin1')
 - ustring.decode('latex+latin1')
where latin1 can be replaced by any other known encoding, also
become available by calling latex.register().

We also make public a dictionary latex_equivalents,
mapping ord(unicode char) to LaTeX code.

D. Eppstein, October 2003.

"""

from __future__ import generators
import codecs
import re


def register():
    """Enable encodings of the form 'latex+x' where x describes another encoding.
    Unicode characters are translated to or from x when possible, otherwise
    expanded to latex.
    """
    codecs.register(_registry)


def getregentry():
    """Encodings module API."""
    return _registry("latex")


def _registry(encoding):
    if encoding == "latex":
        encoding = None
    elif encoding.startswith("latex+"):
        encoding = encoding[6:]
    else:
        return None

    class Codec(codecs.Codec):
        def encode(self, input, errors="strict"):
            """Convert unicode string to latex."""
            output = []
            for c in input:
                if encoding:
                    try:
                        output.append(c.encode(encoding))
                        continue
                    except:
                        pass
                if ord(c) in latex_equivalents:
                    output.append(latex_equivalents[ord(c)])
                else:
                    # pass unicode character straight through
                    # this will be ok in latex if your latex is post-2018
                    # of if you say "\usepackage[utf8]{inputenc}"
                    output += c
                    ## old version used \charXXX in latex
                    ## output += ["{\\char", str(ord(c)), "}"]
            return "".join(output), len(input)

        def decode(self, input, errors="strict"):
            """Convert latex source string to unicode."""
            if encoding:
                input = unicode(input, encoding, errors)

            # Note: we may get buffer objects here.
            # It is not permussable to call join on buffer objects
            # but we can make them joinable by calling unicode.
            # This should always be safe since we are supposed
            # to be producing unicode output anyway.
            x = map(unicode, _unlatex(input))
            return u"".join(x), len(input)

    class StreamWriter(Codec, codecs.StreamWriter):
        pass

    class StreamReader(Codec, codecs.StreamReader):
        pass

    return (Codec().encode, Codec().decode, StreamReader, StreamWriter)


def _tokenize(tex):
    """Convert latex source into sequence of single-token substrings."""
    start = 0
    try:
        # skip quickly across boring stuff
        pos = next(_stoppers.finditer(tex)).span()[0]
    except StopIteration:
        yield tex
        return

    while 1:
        if pos > start:
            yield tex[start:pos]
            if tex[start] == "\\" and not (
                tex[pos - 1].isdigit() and tex[start + 1].isalpha()
            ):
                while pos < len(tex) and tex[pos].isspace():  # skip blanks after csname
                    pos += 1

        while pos < len(tex) and tex[pos] in _ignore:
            pos += 1  # flush control characters
        if pos >= len(tex):
            return
        start = pos
        if tex[pos : pos + 2] in {"$$": None, "/~": None}:  # protect ~ in urls
            pos += 2
        elif tex[pos].isdigit():
            while pos < len(tex) and tex[pos].isdigit():
                pos += 1
        elif tex[pos] == "-":
            while pos < len(tex) and tex[pos] == "-":
                pos += 1
        elif tex[pos] != "\\" or pos == len(tex) - 1:
            pos += 1
        elif not tex[pos + 1].isalpha():
            pos += 2
        else:
            pos += 1
            while pos < len(tex) and tex[pos].isalpha():
                pos += 1
            if tex[start:pos] == "\\char" or tex[start:pos] == "\\accent":
                while pos < len(tex) and tex[pos].isdigit():
                    pos += 1


class _unlatex:
    """Convert tokenized tex into sequence of unicode strings.  Helper for decode()."""

    def __iter__(self):
        """Turn self into an iterator.  It already is one, nothing to do."""
        return self

    def __init__(self, tex):
        """Create a new token converter from a string."""
        self.tex = tuple(_tokenize(tex))  # turn tokens into indexable list
        self.pos = 0  # index of first unprocessed token
        self.lastoutput = "x"  # lastoutput must always be nonempty string

    def __getitem__(self, n):
        """Return token at offset n from current pos."""
        p = self.pos + n
        t = self.tex
        return p < len(t) and t[p] or None

    def next(self):
        """Find and return another piece of converted output."""
        if self.pos >= len(self.tex):
            raise StopIteration
        nextoutput = self.chunk()
        if (
            self.lastoutput[0] == "\\"
            and self.lastoutput[-1].isalpha()
            and nextoutput[0].isalpha()
        ):
            nextoutput = " " + nextoutput  # add extra space to terminate csname
        self.lastoutput = nextoutput
        return nextoutput

    def chunk(self):
        """Grab another set of input tokens and convert them to an output string."""
        for delta, c in self.candidates(0):
            if c in _l2u:
                self.pos += delta
                return unichr(_l2u[c])
            elif len(c) == 2 and c[1] == "i" and (c[0], "\\i") in _l2u:
                self.pos += delta  # correct failure to undot i
                return unichr(_l2u[(c[0], "\\i")])
            elif len(c) == 1 and c[0].startswith("\\char") and c[0][5:].isdigit():
                self.pos += delta
                return unichr(int(c[0][5:]))

        # nothing matches, just pass through token as-is
        self.pos += 1
        return self[-1]

    def candidates(self, offset):
        """Generate pairs delta,c where c is a token or tuple of tokens from tex
        (after deleting extraneous brackets starting at pos) and delta
        is the length of the tokens prior to bracket deletion.
        """
        t = self[offset]
        if t in _blacklist:
            return
        elif t == "{":
            for delta, c in self.candidates(offset + 1):
                if self[offset + delta + 1] == "}":
                    yield delta + 2, c
        elif t == "\\mbox":
            for delta, c in self.candidates(offset + 1):
                yield delta + 1, c
        elif t == "$" and self[offset + 2] == "$":
            yield 3, (t, self[offset + 1], t)
        else:
            q = self[offset + 1]
            if q == "{" and self[offset + 3] == "}":
                yield 4, (t, self[offset + 2])
            elif q:
                yield 2, (t, q)
            yield 1, t


latex_equivalents = {
    0x0023: "{\\#}",
    0x0024: "{\\$}",
    0x0025: "{\\%}",
    0x0026: "{\\&}",
    0x003C: "{\\textless}",
    0x003E: "{\\textgreater}",
    0x005C: "{\\textbackslash}",
    0x005E: "{\\textasciicircum}",
    0x005F: "{\\_}",
    0x007E: "{\\textasciitilde}",
    0x00A0: "~",
    0x00A1: "!`",
    0x00A2: "{\\textcent}",
    0x00A3: "{\\pounds}",
    0x00A7: "{\\S}",
    0x00A8: '{\\"{}}',
    0x00A9: "{\\copyright}",
    0x00AF: "{\\={}}",
    0x00AC: "{\\neg}",
    0x00AD: "{\\-}",
    0x00B0: "{\\mbox{$^\\circ$}}",
    0x00B1: "{\\mbox{$\\pm$}}",
    0x00B2: "{\\mbox{$^2$}}",
    0x00B3: "{\\mbox{$^3$}}",
    0x00B4: "{\\'{}}",
    0x00B5: "{\\mbox{$\\mu$}}",
    0x00B6: "{\\P}",
    0x00B7: "{\\mbox{$\\cdot$}}",
    0x00B8: "{\\c{}}",
    0x00B9: "{\\mbox{$^1$}}",
    0x00BF: "?`",
    0x00C0: "{\\`A}",
    0x00C1: "{\\'A}",
    0x00C2: "{\\^A}",
    0x00C3: "{\\~A}",
    0x00C4: '{\\"A}',
    0x00C5: "{\\AA}",
    0x00C6: "{\\AE}",
    0x00C7: "{\\c{C}}",
    0x00C8: "{\\`E}",
    0x00C9: "{\\'E}",
    0x00CA: "{\\^E}",
    0x00CB: '{\\"E}',
    0x00CC: "{\\`I}",
    0x00CD: "{\\'I}",
    0x00CE: "{\\^I}",
    0x00CF: '{\\"I}',
    0x00D0: "{\\DH}",  # not recognized by purify$
    0x00D1: "{\\~N}",
    0x00D2: "{\\`O}",
    0x00D3: "{\\'O}",
    0x00D4: "{\\^O}",
    0x00D5: "{\\~O}",
    0x00D6: '{\\"O}',
    0x00D7: "{\\mbox{$\\times$}}",
    0x00D8: "{\\O}",
    0x00D9: "{\\`U}",
    0x00DA: "{\\'U}",
    0x00DB: "{\\^U}",
    0x00DC: '{\\"U}',
    0x00DD: "{\\'Y}",
    0x00DE: "{\\TH}",  # not recognized by purify$
    0x00DF: "{\\ss}",
    0x00E0: "{\\`a}",
    0x00E1: "{\\'a}",
    0x00E2: "{\\^a}",
    0x00E3: "{\\~a}",
    0x00E4: '{\\"a}',
    0x00E5: "{\\aa}",
    0x00E6: "{\\ae}",
    0x00E7: "{\\c{c}}",
    0x00E8: "{\\`e}",
    0x00E9: "{\\'e}",
    0x00EA: "{\\^e}",
    0x00EB: '{\\"e}',
    0x00EC: "{\\`\\i}",
    0x00ED: "{\\'\\i}",
    0x00EE: "{\\^\\i}",
    0x00EF: '{\\"\\i}',
    0x00F0: "{\\dh}",  # not recognized by purify$
    0x00F1: "{\\~n}",
    0x00F2: "{\\`o}",
    0x00F3: "{\\'o}",
    0x00F4: "{\\^o}",
    0x00F5: "{\\~o}",
    0x00F6: '{\\"o}',
    0x00F7: "{\\mbox{$\\div$}}",
    0x00F8: "{\\o}",
    0x00F9: "{\\`u}",
    0x00FA: "{\\'u}",
    0x00FB: "{\\^u}",
    0x00FC: '{\\"u}',
    0x00FD: "{\\'y}",
    0x00FE: "{\\th}",  # not recognized by purify$
    0x00FF: '{\\"y}',
    0x0100: "{\\=A}",
    0x0101: "{\\=a}",
    0x0102: "{\\u{A}}",
    0x0103: "{\\u{a}}",
    0x0104: "{\\k{A}}",
    0x0105: "{\\k{a}}",
    0x0106: "{\\'C}",
    0x0107: "{\\'c}",
    0x0108: "{\\^C}",
    0x0109: "{\\^c}",
    0x010A: "{\\.C}",
    0x010B: "{\\.c}",
    0x010C: "{\\v{C}}",
    0x010D: "{\\v{c}}",
    0x010E: "{\\v{D}}",
    0x010F: "{\\v{d}}",
    0x0110: "{\\DJ}",  # not recognized by purify$ (Vietnamese, Serb, Croat)
    0x0111: "{\\dj}",  # not recognized by purify$ (Vietnamese, Serb, Croat)
    0x0112: "{\\=E}",
    0x0113: "{\\=e}",
    0x0114: "{\\u{E}}",
    0x0115: "{\\u{e}}",
    0x0116: "{\\.E}",
    0x0117: "{\\.e}",
    0x0118: "{\\k{E}}",
    0x0119: "{\\k{e}}",
    0x011A: "{\\v{E}}",
    0x011B: "{\\v{e}}",
    0x011C: "{\\^G}",
    0x011D: "{\\^g}",
    0x011E: "{\\u{G}}",
    0x011F: "{\\u{g}}",
    0x0120: "{\\.G}",
    0x0121: "{\\.g}",
    0x0122: "{\\c{G}}",
    0x0123: "{\\c{g}}",
    0x0124: "{\\^H}",
    0x0125: "{\\^h}",
    0x0128: "{\\~I}",
    0x0129: "{\\~\\i}",
    0x012A: "{\\=I}",
    0x012B: "{\\=\\i}",
    0x012C: "{\\u{I}}",
    0x012D: "{\\u\\i}",
    0x012E: "{\\k{I}}",
    0x012F: "{\\k{i}}",
    0x0130: "{\\.I}",
    0x0131: "{\\i}",
    0x0132: "{IJ}",
    0x0133: "{ij}",
    0x0134: "{\\^J}",
    0x0135: "{\\^\\j}",
    0x0136: "{\\c{K}}",
    0x0137: "{\\c{k}}",
    0x0139: "{\\'L}",
    0x013A: "{\\'l}",
    0x013B: "{\\c{L}}",
    0x013C: "{\\c{l}}",
    0x013D: "{\\v{L}}",
    0x013E: "{\\v{l}}",
    0x0141: "{\\L}",
    0x0142: "{\\l}",
    0x0143: "{\\'N}",
    0x0144: "{\\'n}",
    0x0145: "{\\c{N}}",
    0x0146: "{\\c{n}}",
    0x0147: "{\\v{N}}",
    0x0148: "{\\v{n}}",
    0x014C: "{\\=O}",
    0x014D: "{\\=o}",
    0x014E: "{\\u{O}}",
    0x014F: "{\\u{o}}",
    0x0150: "{\\H{O}}",
    0x0151: "{\\H{o}}",
    0x0152: "{\\OE}",
    0x0153: "{\\oe}",
    0x0154: "{\\'R}",
    0x0155: "{\\'r}",
    0x0156: "{\\c{R}}",
    0x0157: "{\\c{r}}",
    0x0158: "{\\v{R}}",
    0x0159: "{\\v{r}}",
    0x015A: "{\\'S}",
    0x015B: "{\\'s}",
    0x015C: "{\\^S}",
    0x015D: "{\\^s}",
    0x015E: "{\\c{S}}",
    0x015F: "{\\c{s}}",
    0x0160: "{\\v{S}}",
    0x0161: "{\\v{s}}",
    0x0162: "{\\c{T}}",
    0x0163: "{\\c{t}}",
    0x0164: "{\\v{T}}",
    0x0165: "{\\v{t}}",
    0x0168: "{\\~U}",
    0x0169: "{\\~u}",
    0x016A: "{\\=U}",
    0x016B: "{\\=u}",
    0x016C: "{\\u{U}}",
    0x016D: "{\\u{u}}",
    0x016E: "{\\r{U}}",
    0x016F: "{\\r{u}}",
    0x0170: "{\\H{U}}",
    0x0171: "{\\H{u}}",
    0x0172: "{\\k{U}}",
    0x0173: "{\\k{u}}",
    0x0174: "{\\^W}",
    0x0175: "{\\^w}",
    0x0176: "{\\^Y}",
    0x0177: "{\\^y}",
    0x0178: '{\\"Y}',
    0x0179: "{\\'Z}",
    0x017A: "{\\'z}",
    0x017B: "{\\.Z}",
    0x017C: "{\\.z}",
    0x017D: "{\\v{Z}}",
    0x017E: "{\\v{z}}",
    0x01C4: "{D\\v{Z}}",
    0x01C5: "{D\\v{z}}",
    0x01C6: "{d\\v{z}}",
    0x01C7: "{LJ}",
    0x01C8: "{Lj}",
    0x01C9: "{lj}",
    0x01CA: "{NJ}",
    0x01CB: "{Nj}",
    0x01CC: "{nj}",
    0x01CD: "{\\v{A}}",
    0x01CE: "{\\v{a}}",
    0x01CF: "{\\v{I}}",
    0x01D0: "{\\v\\i}",
    0x01D1: "{\\v{O}}",
    0x01D2: "{\\v{o}}",
    0x01D3: "{\\v{U}}",
    0x01D4: "{\\v{u}}",
    0x01E6: "{\\v{G}}",
    0x01E7: "{\\v{g}}",
    0x01E8: "{\\v{K}}",
    0x01E9: "{\\v{k}}",
    0x01EA: "{\\k{O}}",
    0x01EB: "{\\k{o}}",
    0x01F0: "{\\v\\j}",
    0x01F1: "{DZ}",
    0x01F2: "{Dz}",
    0x01F3: "{dz}",
    0x01F4: "{\\'G}",
    0x01F5: "{\\'g}",
    0x01FC: "{\\'\\AE}",
    0x01FD: "{\\'\\ae}",
    0x01FE: "{\\'\\O}",
    0x01FF: "{\\'\\o}",
    0x0218: "{\\textcommabelow{S}}",  # Romanian
    0x0219: "{\\textcommabelow{s}}",  # Romanian
    0x021A: "{\\textcommabelow{T}}",  # Romanian
    0x021B: "{\\textcommabelow{t}}",  # Romanian
    0x02C6: "{\\^{}}",
    0x02DC: "{\\~{}}",
    0x02D8: "{\\u{}}",
    0x02D9: "{\\.{}}",
    0x02DA: "{\\r{}}",
    0x02DD: "{\\H{}}",
    0x02DB: "{\\k{}}",
    0x02C7: "{\\v{}}",
    0x03C0: "{\\mbox{$\\pi$}}",
    # consider adding more Greek here
    0xFB01: "{fi}",
    0xFB02: "{fl}",
    0x2013: "{--}",
    0x2014: "{---}",
    0x2018: "{`}",
    0x2019: "{'}",
    0x201C: "{``}",
    0x201D: "{''}",
    0x2020: "{\\dag}",
    0x2021: "{\\ddag}",
    0x2122: "{\\mbox{$^\\mbox{TM}$}}",
    0x2022: "{\\mbox{$\\bullet$}}",
    0x2026: "{\\ldots}",
    0x2202: "{\\mbox{$\\partial$}}",
    0x220F: "{\\mbox{$\\prod$}}",
    0x2211: "{\\mbox{$\\sum$}}",
    0x221A: "{\\mbox{$\\surd$}}",
    0x221E: "{\\mbox{$\\infty$}}",
    0x222B: "{\\mbox{$\\int$}}",
    0x2248: "{\\mbox{$\\approx$}}",
    0x2260: "{\\mbox{$\\neq$}}",
    0x2264: "{\\mbox{$\\leq$}}",
    0x2265: "{\\mbox{$\\geq$}}",
}
for _i in range(0x0020):
    if _i not in latex_equivalents:
        latex_equivalents[_i] = ""
for _i in range(0x0020, 0x007F):
    if _i not in latex_equivalents:
        latex_equivalents[_i] = chr(_i)

# Characters that should be ignored and not output in tokenization
_ignore = set([chr(i) for i in list(range(32)) + [127]]) - set("\t\n\r")

# Regexp of chars not in blacklist, for quick start of tokenize
_stoppers = re.compile("[\x00-\x1f!$\\-?\\{~\\\\`']")

_blacklist = set(" \n\r")
_blacklist.add(None)  # shortcut candidate generation at end of data

# Construction of inverse translation table
_l2u = {"\ ": ord(" ")}  # unexpanding space makes no sense in non-TeX contexts

for _tex in latex_equivalents:
    if _tex <= 0x0020 or (_tex <= 0x007F and len(latex_equivalents[_tex]) <= 1):
        continue  # boring entry
    _toks = tuple(_tokenize(latex_equivalents[_tex]))
    if _toks[0] == "{" and _toks[-1] == "}":
        _toks = _toks[1:-1]
    if _toks[0].isalpha():
        continue  # don't turn ligatures into single chars
    if len(_toks) == 1 and (_toks[0] == "'" or _toks[0] == "`"):
        continue  # don't turn ascii quotes into curly quotes
    if _toks[0] == "\\mbox" and _toks[1] == "{" and _toks[-1] == "}":
        _toks = _toks[2:-1]
    if len(_toks) == 4 and _toks[1] == "{" and _toks[3] == "}":
        _toks = (_toks[0], _toks[2])
    if len(_toks) == 1:
        _toks = _toks[0]
    _l2u[_toks] = _tex

# Shortcut candidate generation for certain useless candidates:
# a character is in _blacklist if it can not be at the start
# of any translation in _l2u.  We use this to quickly skip through
# such characters before getting to more difficult-translate parts.
# _blacklist is defined several lines up from here because it must
# be defined in order to call _tokenize, however it is safe to
# delay filling it out until now.

for i in range(0x0020, 0x007F):
    _blacklist.add(chr(i))
_blacklist.remove("{")
_blacklist.remove("$")
for candidate in _l2u:
    if isinstance(candidate, tuple):
        if not candidate or not candidate[0]:
            continue
        firstchar = candidate[0][0]
    else:
        firstchar = candidate[0]
    _blacklist.discard(firstchar)

if __name__ == "__main__":
    import fileinput

    register()
    for line in fileinput.input():
        print(codecs.encode(line.rstrip(), "latex"))


# bin/anthology/papers.py

# -*- coding: utf-8 -*-
#
# Copyright 2019-2022 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import cached_property
import iso639
import logging as log

from .utils import (
    build_anthology_id,
    parse_element,
    infer_url,
    infer_attachment_url,
    remove_extra_whitespace,
    is_journal,
    is_volume_id,
)
from . import data

# For bibliography export
from .formatter import (
    bibtex_encode,
    bibtex_make_entry,
    CiteprocFormatter,
    MarkupFormatter,
)


class Paper:
    def __init__(self, paper_id, ingest_date, volume, formatter=None, venue_index=None):
        self.parent_volume = volume
        if formatter is None:
            formatter = MarkupFormatter()
        self.formatter = formatter
        self.venue_index = venue_index
        self._id = paper_id
        self._ingest_date = ingest_date
        self._bibkey = None
        self._citeproc_json = None
        self.is_volume = paper_id == "0"

        # initialize metadata with keys inherited from volume
        self.attrib = {}
        for key, value in volume.attrib.items():
            # Only inherit 'editor' for frontmatter
            if (key == "editor" and not self.is_volume) or key in (
                "collection_id",
                "booktitle",
                "id",
                "meta_data",
                "meta_journal_title",
                "meta_volume",
                "meta_issue",
                "sigs",
                "venues",
                "meta_date",
                "url",
                "pdf",
                "xml_url",
            ):
                continue

            self.attrib[key] = value

    @cached_property
    def url(self):
        # If <url> field not present, use ID.
        # But see https://github.com/acl-org/acl-anthology/issues/997.
        return infer_url(self.attrib.get("xml_url", self.full_id))

    @cached_property
    def pdf(self):
        url = self.attrib.get("xml_url", None)
        if url is not None:
            return infer_url(url, template=data.PDF_LOCATION_TEMPLATE)
        return None

    @cached_property
    def videos(self):
        videos = self.attrib.get("video", None)
        if videos:
            return [
                infer_url(video, template=data.VIDEO_LOCATION_TEMPLATE)
                for video in videos
            ]
        return []

    def _parse_revision_or_errata(self, tag):
        for item in self.attrib.get(tag, []):
            # Expand URLs with paper ID
            if not item["url"].startswith(self.full_id):
                log.error(
                    "{} must begin with paper ID '{}', but is '{}'".format(
                        tag, self.full_id, item["url"]
                    )
                )
            item["url"] = data.PDF_LOCATION_TEMPLATE.format(item["url"])
        return self.attrib.get(tag, [])

    @cached_property
    def revisions(self):
        return self._parse_revision_or_errata("revision")

    @cached_property
    def errata(self):
        return self._parse_revision_or_errata("erratum")

    @cached_property
    def attachments(self):
        for item in self.attrib.get("attachment", []):
            item["url"] = infer_attachment_url(item["url"], self.full_id)
        return self.attrib.get("attachment", [])

    @cached_property
    def thumbnail(self):
        return data.PDF_THUMBNAIL_LOCATION_TEMPLATE.format(self.full_id)

    @cached_property
    def title(self):
        return self.get_title("plain")

    @cached_property
    def booktitle(self):
        return self.get_booktitle("plain")

    def from_xml(xml_element, *args):
        ingest_date = xml_element.get("ingest-date", data.UNKNOWN_INGEST_DATE)

        # Default to paper ID "0" (for front matter)
        paper = Paper(xml_element.get("id", "0"), ingest_date, *args)

        # Set values from parsing the XML element (overwriting
        # and changing some initialized from the volume metadata)
        for key, value in parse_element(xml_element).items():
            if key == "author" and "editor" in paper.attrib:
                del paper.attrib["editor"]
            if key == "bibkey":
                paper.bibkey = value
            else:
                paper.attrib[key] = value

        # Frontmatter title is the volume 'booktitle'
        if paper.is_volume:
            paper.attrib["xml_title"] = paper.attrib["xml_booktitle"]
            paper.attrib["xml_title"].tag = "title"

        # Remove booktitle for frontmatter and journals
        if paper.is_volume or is_journal(paper.full_id):
            del paper.attrib["xml_booktitle"]

        if "editor" in paper.attrib:
            if paper.is_volume:
                if "author" in paper.attrib:
                    log.warn(
                        f"Paper {paper.full_id} has both <editor> and <author>; ignoring <author>"
                    )
                # Proceedings editors are considered authors for their front matter
                paper.attrib["author"] = paper.attrib["editor"]
                del paper.attrib["editor"]
            else:
                log.warn(
                    f"Paper {paper.full_id} has <editor> but is not a proceedings volume; ignoring <editor>"
                )
        if "pages" in paper.attrib:
            if paper.attrib["pages"] is not None:
                paper._interpret_pages()
            else:
                del paper.attrib["pages"]

        if "author" in paper.attrib:
            paper.attrib["author_string"] = ", ".join(
                [x[0].full for x in paper.attrib["author"]]
            )

        paper.attrib["citation"] = paper.as_markdown()

        # An empty value gets set to None, which causes hugo to skip it over
        # entirely. Set it here to a single space, instead. There's probably
        # a better way to do this.
        if "retracted" in paper.attrib and paper.attrib["retracted"] is None:
            paper.attrib["retracted"] = " "

        if "removed" in paper.attrib and paper.attrib["removed"] is None:
            paper.attrib["removed"] = " "

        return paper

    def _interpret_pages(self):
        """Splits up 'pages' field into first and last page, if possible.

        This is used for metadata in the generated HTML."""
        for s in ("--", "-", "–"):
            if self.attrib["pages"].count(s) == 1:
                self.attrib["page_first"], self.attrib["page_last"] = self.attrib[
                    "pages"
                ].split(s)
                self.attrib["pages"] = self.attrib["pages"].replace(s, "–")
                return

    @property
    def ingest_date(self):
        """Inherit publication date from parent, but self overrides. May be undefined."""
        if self._ingest_date:
            return self._ingest_date
        if self.parent_volume:
            return self.parent_volume.ingest_date
        return data.UNKNOWN_INGEST_DATE

    @property
    def collection_id(self):
        return self.parent_volume.collection_id

    @property
    def volume_id(self):
        return self.parent_volume.volume_id

    @property
    def paper_id(self):
        return self._id

    @property
    def full_id(self):
        return self.anthology_id

    @cached_property
    def anthology_id(self):
        return build_anthology_id(self.collection_id, self.volume_id, self.paper_id)

    @property
    def bibkey(self):
        return self._bibkey

    @bibkey.setter
    def bibkey(self, value):
        self._bibkey = value

    @property
    def bibtype(self):
        """Return the BibTeX entry type for this paper."""
        if is_journal(self.full_id):
            return "article"
        elif self.is_volume:
            return "proceedings"
        else:
            return "inproceedings"

    @property
    def csltype(self):
        """Return the CSL type for this paper.

        cf. https://docs.citationstyles.org/en/stable/specification.html#appendix-iii-types
        """
        if is_journal(self.full_id):
            return "article-journal"
        elif self.is_volume:
            return "book"
        else:
            return "paper-conference"

    @property
    def parent_volume_id(self):
        if self.parent_volume is not None:
            return self.parent_volume.full_id
        return None

    @property
    def has_abstract(self):
        return "xml_abstract" in self.attrib

    @property
    def is_retracted(self) -> bool:
        return "retracted" in self.attrib

    @property
    def is_removed(self) -> bool:
        return "removed" in self.attrib

    @property
    def isbn(self):
        return self.attrib.get("isbn", None)

    @property
    def langcode(self):
        """Returns the ISO-639 language code, if present"""
        return self.attrib.get("language", None)

    @property
    def language(self):
        """Returns the language name, if present"""
        lang = None
        if self.langcode:
            lang = iso639.languages.get(part3=self.langcode).name
        return lang

    def get(self, name, default=None):
        try:
            return self.attrib[name]
        except KeyError:
            return default

    def get_title(self, form="xml"):
        """Returns the paper title, optionally formatting it.

        Accepted formats:
          - xml:   Include any contained XML tags unchanged
          - plain: Strip all XML tags, returning only plain text
          - html:  Convert XML tags into valid HTML tags
          - latex: Convert XML tags into LaTeX commands
        """
        return self.formatter(self.get("xml_title"), form)

    def get_abstract(self, form="xml"):
        """Returns the abstract, optionally formatting it.

        See `get_title()` for details.
        """
        return self.formatter(self.get("xml_abstract"), form, allow_url=True)

    def get_booktitle(self, form="xml", default=""):
        """Returns the booktitle, optionally formatting it.

        See `get_title()` for details.
        """
        if "xml_booktitle" in self.attrib:
            return self.formatter(self.get("xml_booktitle"), form)
        elif self.parent_volume is not None:
            return self.parent_volume.get("title")
        else:
            return default

    def as_bibtex(self, concise=False):
        """Return the BibTeX entry for this paper."""
        # Build BibTeX entry
        bibkey = self.bibkey
        bibtype = self.bibtype
        entries = [("title", self.get_title(form="latex"))]
        for people in ("author", "editor"):
            if people in self.attrib:
                entries.append(
                    (people, "  and  ".join(p.as_bibtex() for p, _ in self.get(people)))
                )
        if is_journal(self.full_id):
            entries.append(
                ("journal", bibtex_encode(self.parent_volume.get("meta_journal_title")))
            )
            journal_volume = self.parent_volume.get(
                "meta_volume", self.parent_volume.get("volume")
            )
            if journal_volume:
                entries.append(("volume", journal_volume))
            journal_issue = self.parent_volume.get(
                "meta_issue", self.parent_volume.get("issue")
            )
            if journal_issue:
                entries.append(("number", journal_issue))
        else:
            # not is_journal(self.full_id)
            if "xml_booktitle" in self.attrib:
                entries.append(("booktitle", self.get_booktitle(form="latex")))
            elif bibtype != "proceedings":
                entries.append(("booktitle", self.parent_volume.get_title(form="latex")))
        for entry in ("month", "year", "address", "publisher", "note"):
            if self.get(entry) is not None:
                entries.append((entry, bibtex_encode(self.get(entry))))
        if self.url is not None:
            entries.append(("url", self.url))
        if "doi" in self.attrib:
            # don't want latex escapes such as
            # doi = "10.1162/coli{\_}a{\_}00008",
            entries.append(("doi", self.get("doi")))
        if "pages" in self.attrib:
            entries.append(("pages", self.get("pages").replace("–", "--")))
        if "xml_abstract" in self.attrib and not concise:
            entries.append(("abstract", self.get_abstract(form="latex")))
        if self.language:
            entries.append(("language", self.language))
        if self.isbn:
            entries.append(("ISBN", self.isbn))

        # Serialize it
        return bibtex_make_entry(bibkey, bibtype, entries)

    def as_citeproc_json(self):
        """Return a citation object suitable for CiteProcJSON."""
        if self._citeproc_json is None:
            data = {
                "id": self.bibkey,
                "title": self.get_title(form="text"),
                "type": self.csltype,
            }
            if "author" in self.attrib:
                data["author"] = [p.as_citeproc_json() for p, _ in self.get("author")]
            if "editor" in self.attrib:
                # or should this be "container-author"/"collection-editor" here?
                data["editor"] = [p.as_citeproc_json() for p, _ in self.get("editor")]
            if is_journal(self.full_id):
                data["container-title"] = self.parent_volume.get("meta_journal_title")
                journal_volume = self.parent_volume.get(
                    "meta_volume", self.parent_volume.get("volume")
                )
                if journal_volume:
                    data["volume"] = journal_volume
                journal_issue = self.parent_volume.get(
                    "meta_issue", self.parent_volume.get("issue")
                )
                if journal_issue:
                    data["issue"] = journal_issue
            else:
                if "xml_booktitle" in self.attrib:
                    data["container-title"] = self.get_booktitle(form="text")
                elif self.bibtype != "proceedings":
                    data["container-title"] = self.parent_volume.get_title(form="text")
            data["publisher"] = self.get("publisher", "")
            data["publisher-place"] = self.get("address", "")
            data["issued"] = {
                "date-parts": [
                    [self.get("year")]
                ]  # TODO: month needs to be a numeral to be included
            }
            data["URL"] = self.url
            if "doi" in self.attrib:
                data["DOI"] = self.get("doi")
            if "pages" in self.attrib:
                data["page"] = self.get("pages")
            if self.isbn:
                data["ISBN"] = self.isbn
            self._citeproc_json = [data]
        return self._citeproc_json

    def as_citation_html(
        self, style="association-for-computational-linguistics", link_title=True
    ):
        html = CiteprocFormatter.render_html_citation(self, style)
        if link_title:
            # It would be nicer to do this within Citeproc, which would probably
            # entail writing/updating our own CSL style.
            title = self.get_title("plain")
            link = f'<a href="{self.url}">{title}</a>'
            html = html.replace(title, link)
        return html

    def as_markdown(self, concise=False):
        """Return a Markdown-formatted entry."""
        title = self.get_title(form="text")

        authors = ""
        for field in ("author", "editor"):
            if field in self.attrib:
                people = [person[0] for person in self.get(field)]
                num_people = len(people)
                if num_people == 1:
                    authors = people[0].last
                elif num_people == 2:
                    authors = f"{people[0].last} & {people[1].last}"
                elif num_people >= 3:
                    authors = f"{people[0].last} et al."

        year = self.get("year")
        venue = self.venue_index.get_main_venue(self.parent_volume_id)
        url = self.url

        # hard-coded exception for old-style W-* volumes without an annotated
        # main venue
        if venue != "WS":
            return f"[{title}]({url}) ({authors}, {venue} {year})"
        return f"[{title}]({url}) ({authors}, {year})"

    def as_dict(self):
        value = self.attrib.copy()
        value["paper_id"] = self.paper_id
        value["parent_volume_id"] = self.parent_volume_id
        value["bibkey"] = self.bibkey
        value["bibtype"] = self.bibtype
        value["language"] = self.language
        value["url"] = self.url
        value["title"] = self.title
        value["booktitle"] = self.booktitle
        if self.pdf:
            value["pdf"] = self.pdf
        if self.revisions:
            value["revision"] = self.revisions
        if self.errata:
            value["erratum"] = self.errata
        if self.videos:
            value["video"] = self.videos
        if self.attachments:
            value["attachment"] = self.attachments
        value["thumbnail"] = self.thumbnail
        return value

    def items(self):
        return self.attrib.items()

    def iter_people(self):
        for role in ("author", "editor"):
            for name, id_ in self.get(role, []):
                yield (name, id_, role)


# bin/anthology/people.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging as log
from functools import cached_property
import re
from slugify import slugify
import anthology.formatter as my_formatter


def score_variant(name):
    """Heuristically assign scores to names, with the idea of assigning higher
    scores to spellings more likely to be the correct canonical variant."""
    name = repr(name)
    # Prefer longer variants
    score = len(name)
    # Prefer variants with non-ASCII characters
    score += sum((ord(c) > 127) for c in name)
    # Penalize upper-case characters after word boundaries
    score -= sum(any(c.isupper() for c in w[1:]) for w in re.split(r"\W+", name))
    # Penalize lower-case characters at word boundaries
    score -= sum(w[0].islower() if w else 0 for w in re.split(r"\W+", name))
    if name[0].islower():  # extra penalty for first name
        score -= 1

    return score


class PersonName:
    first, last = "", ""

    def __init__(self, first, last, script="roman", variant: "PersonName" = None):
        self.first = first if first is not None else ""
        self.last = last
        self.script = script
        self.variant = variant

    def from_element(person_element):
        """
        Reads from the XML, which includes an optional first name, a last name,
        and an optional variant (itself containing an optional first name, and a
        last name).
        """
        first, last = "", ""
        # The name variant script, defaults to roman
        script = person_element.attrib.get("script", "roman")
        variant = None
        for element in person_element:
            tag = element.tag

            # These are guaranteed to occur at most once by the schema
            if tag == "first":
                first = element.text or ""
            elif tag == "last":
                last = element.text or ""
            elif tag == "variant":
                variant = PersonName.from_element(element)

        return PersonName(first, last, script=script, variant=variant)

    def from_repr(repr_):
        parts = repr_.split(" || ")
        if len(parts) > 1:
            first, last = parts[0], parts[1]
        else:
            first, last = "", parts[0]
        return PersonName(first, last)

    def from_dict(dict_):
        first = dict_.get("first", "")
        if first is None:
            first = ""
        last = dict_["last"]
        return PersonName(first, last)

    @cached_property
    def full(self):
        """
        Return the full rendering of the name.
        This includes any name variant in parentheses.
        Currently handles both Roman and Han scripts.
        """
        if self.script.startswith("han"):
            form = f"{self.last}{self.first}"
        else:  # default to "roman"
            form = f"{self.first} {self.last}"

        if self.variant is not None:
            return f"{form} ({self.variant.full})"
        else:
            return form

    @cached_property
    def slug(self):
        # This is effectively used as the person's "id".
        # If two names slugify to the same thing, we assume they are the same person.
        # This happens when there are missing accents in one version, or
        # when we have an inconsistent first/last split for multiword names.
        # These cases have in practice always referred to the same person.
        slug = slugify(repr(self))
        if not slug:
            slug = "none"
        return slug

    @cached_property
    def score(self):
        return score_variant(self)

    @property
    def id_(self):
        return repr(self)

    def as_bibtex(self):
        if not self.first:
            return f"{{{my_formatter.bibtex_encode(self.last)}}}"
        return my_formatter.bibtex_encode(f"{self.last}, {self.first}")

    def as_citeproc_json(self):
        if not self.first:
            return {"family": self.last}
        return {"family": self.last, "given": self.first}

    def as_dict(self):
        return {"first": self.first, "last": self.last, "full": self.full}

    def without_variant(self):
        if self.variant is None:
            return self
        return PersonName(self.first, self.last)

    def __eq__(self, other):
        if other is None:
            return False
        return (
            (self.first == other.first)
            and (self.last == other.last)
            and (self.script == other.script)
            and (self.variant == other.variant)
        )

    def __lt__(self, other):
        return self.full < other.full

    def __str__(self):
        return self.full

    def __repr__(self):
        if not self.first:
            return self.last
        return f"{self.first} || {self.last}"

    def __hash__(self):
        return hash(self.full)


# bin/anthology/sigs.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict, namedtuple
from glob import glob
from slugify import slugify
import logging as log
import yaml

from anthology.utils import deconstruct_anthology_id, infer_year

try:
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader


SIGEvent = namedtuple(
    "SIGEvent", ["anthology_id", "name", "url", "year"], defaults=[None, None, None, None]
)


def _sigevent_to_repr(event):
    if event.anthology_id is not None:
        return event.anthology_id
    return {"name": event.name, "url": event.url}


class SIGIndex:
    def __init__(self, srcdir=None):
        self.sigs = {}
        if srcdir is not None:
            self.load_from_dir(srcdir)

    def load_from_dir(self, sigdir):
        self.directory = sigdir
        for filename in glob(f"{self.directory}/yaml/sigs/*.yaml"):
            log.debug(f"Instantiating SIG from {filename}...")
            with open(filename, "r") as f:
                data = yaml.load(f, Loader=Loader)
                sig = SIG.from_dict(data)
                self.sigs[sig.acronym] = sig

    def add_volume(self, signame, volume):
        """
        Adds a volume to a SIG.
        """
        self.sigs[signame.upper()].add_volume(volume)

    def dump(self):
        """
        Dumps the venue database to file.
        """
        for acronym, sig in self.items():
            with open(f"{self.directory}/yaml/sigs/{acronym.lower()}.yaml", "wt") as f:
                print(yaml.dump(sig._data, sort_keys=False, allow_unicode=True), file=f)

    def remove_volume(self, full_volume_id):
        for acronym, sig in self.sigs.items():
            sig.remove_volume(full_volume_id)

    def get_associated_sigs(self, anthology_id):
        return [
            acronym
            for acronym, sig in self.sigs.items()
            if sig.is_associated_with(anthology_id)
        ]

    def items(self):
        return self.sigs.items()


class SIG:
    def __init__(self, acronym, name, url):
        self.acronym = acronym
        self.name = name
        self.url = url
        self._data = {}
        self._associated_events = []
        self.events_by_year = {}

    def from_dict(dict_):
        sig = SIG(dict_["ShortName"], dict_["Name"], dict_.get("URL", None))
        sig.data = dict_
        return sig

    def remove_volume(self, full_volume_id):
        """
        Volumes with future ingestion dates are not built and may need to be removed from a SIG's listing.

        `full_volume_id` looks like `P19-1` or `W19-31`
        """
        collection_id, _, _ = deconstruct_anthology_id(full_volume_id)
        year = int(infer_year(collection_id))
        if year in self.events_by_year:
            self.events_by_year[year] = [
                event
                for event in self.events_by_year[year]
                if not event[0] == full_volume_id
            ]

    def add_volume(self, full_volume_id):
        collection_id, _, _ = deconstruct_anthology_id(full_volume_id)
        year = int(infer_year(collection_id))
        if not year in self.events_by_year:
            self.events_by_year[year] = []
        self.events_by_year[year].append(full_volume_id)

    @property
    def associated_events(self):
        return self._associated_events

    @property
    def data(self):
        return self._data

    @data.setter
    def data(self, dict_):
        self._data = dict_
        self._associated_events = []
        self.events_by_year = defaultdict(list)
        for eventdicts in dict_["Meetings"]:
            for year, events in eventdicts.items():
                for event in events:
                    if isinstance(event, str):
                        ev = SIGEvent(anthology_id=event, year=year)
                    elif isinstance(event, dict):
                        ev = SIGEvent(
                            name=event["Name"], url=event.get("URL", None), year=year
                        )
                    else:
                        log.warning(
                            f"In SIG '{self.acronym}': Unknown event format: {type(event)}"
                        )
                    self._associated_events.append(ev)
                    self.events_by_year[year].append(ev)

    @property
    def slug(self):
        return slugify(self.acronym)

    @property
    def volumes_by_year(self):
        return {
            y: [_sigevent_to_repr(e) for e in k] for y, k in self.events_by_year.items()
        }

    @property
    def years(self):
        return self.events_by_year.keys()

    def is_associated_with(self, anthology_id):
        return any(ev.anthology_id == anthology_id for ev in self._associated_events)


# bin/anthology/texmath.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from lxml import etree
import csv
import logging as log
import os
from TexSoup import TexSoup
from TexSoup.data import TexCmd, TexText, TexGroup

FUNCTION_NAMES = ("lim", "log")


def _append_text(text, trg):
    if not text:
        return
    if len(trg):
        if trg[-1].tail is not None:
            trg[-1].tail += text
        else:
            trg[-1].tail = text
    else:
        if trg.text is not None:
            trg.text += text
        else:
            trg.text = text


class TexMath:
    """Interpreter and converter for TeX inline math expressions.

    This class uses TexSoup (https://github.com/alvinwan/TexSoup) to parse a TeX
    expression and converts it to valid HTML.  The conversion combines a small
    number of handwritten rules with a mapping of LaTeX math mode commands to
    Unicode symbols (http://milde.users.sourceforge.net/LUCR/Math/).  Parts that
    cannot be interpreted using this simple method are preserved as raw LaTeX.
    """

    def __init__(self, symbolsfile=None):
        self.cmd_map = {}
        if symbolsfile is None:
            symbolsfile = os.path.join(
                os.path.dirname(os.path.abspath(__file__)), "unimathsymbols.txt"
            )
        self.load_symbols(symbolsfile)

    def load_symbols(self, filename):
        with open(filename, "r", encoding="utf8") as f:
            reader = csv.reader(f, delimiter="^")
            for row in reader:
                if row[0].startswith("#"):  # comment
                    continue
                assert len(row) == 8, "Expect eight-column format"
                char, cmd = row[1], row[2]
                if cmd.startswith("\\"):
                    self.cmd_map[cmd[1:]] = char
                if row[-1].startswith("= ") and ", " in row[-1]:
                    # last column sometimes contains alternative command
                    cmd = row[-1][2:].split(", ")[0]
                    if cmd.startswith("\\"):
                        self.cmd_map[cmd[1:]] = char

    def _parse(self, everything, trg):
        """Parses a list of TeX constituents into an lxml.etree._Element.

        Arguments:
            everything: An iterator over TeX constituents as provided by TexSoup
            trg: The lxml.etree._Element to parse the expression into

        The approach of iterating over the TeX constituents roughly follows
        <https://github.com/alvinwan/TexSoup/blob/master/examples/list_everything.py>.
        """
        sxscript = False  # Tracks whether we're in a subscript/superscript
        for code in everything:
            if isinstance(code, TexCmd):
                # code is a TeX command
                self._parse_command(code, trg)
            elif isinstance(code, TexText) or isinstance(code, str):
                # code is text
                sxscript = self._parse_text(code, trg)
            elif isinstance(code, TexGroup):
                # If in subscript/superscript, wrap the entire element in respective tag
                if sxscript:
                    my_trg = etree.Element(sxscript)
                    self._parse(code.contents, my_trg)
                    trg.append(my_trg)
                    sxscript = False
                # Otherwise, just parse it normally
                else:
                    self._parse(code.contents, trg)
            else:
                log.error(f"TeX-math parser got unhandled element: {type(code)}")

    def _parse_command(self, code, trg):
        args = list(code.args)
        name = str(code.name)
        # Check if the command is in the list of known Unicode mappings
        if name in self.cmd_map:
            _append_text(self.cmd_map[name], trg)
            self._parse(args, trg)
        # Check if command + arguments is in the list of known Unicode mappings
        # (this covers commands like "\mathcal{A}", which have their own entries)
        elif str(code)[1:] in self.cmd_map:
            _append_text(self.cmd_map[str(code)[1:]], trg)
        # Check if command is a known function name (e.g. "log")
        elif name in FUNCTION_NAMES:
            sx = etree.Element("span")
            sx.attrib["class"] = "tex-math-function"
            sx.text = str(name)
            trg.append(sx)
            self._parse(args, trg)
        # Handle fractions
        elif name == "frac":
            self._parse_fraction(args, trg)
        # Handle \textrm (-- currently does nothing)
        elif name == "textrm":
            sx = etree.Element("span")
            self._parse(args, sx)
            trg.append(sx)
        # Handle stuff that should be displayed bolder
        elif name in ("mathbf", "boldsymbol"):
            sx = etree.Element("strong")
            self._parse(args, sx)
            trg.append(sx)
        # Give up, but preserve element
        else:
            log.warn(f"Unknown TeX-math command: {code}")
            self._append_unparsed(code, trg)

    def _parse_fraction(self, args, trg):
        if len(args) != 2:
            log.warn(f"Couldn't parse \\frac: got {len(args)} arguments, expected 2")
            self._append_unparsed(code, trg)
        else:
            # Represent numerator of fraction as superscript
            sx = etree.Element("sup")
            self._parse([args[0]], sx)
            trg.append(sx)
            # Unicode symbol for fraction slash
            _append_text("\u2044", trg)
            # Represent denominator of fraction as subscript
            sx = etree.Element("sub")
            self._parse([args[1]], sx)
            trg.append(sx)

    def _parse_text(self, code, trg):
        text = str(code)
        # TexSoup doesn't parse any non-alpha command as a command. Ex: \$
        # However it does seperate them into their own text part. Ex: 'r\\&dd' -> ['r', '\\&', 'dd']
        # Therefore try to do command mapping replacement of all text beginning with \ and of length 2
        if len(text) == 2 and text[0] == '\\':
            text = self.cmd_map.get(text[1], text)
            _append_text(text, trg)
            return
        # parse ^ and _ (won't get recognized as separate nodes by TexSoup)
        sxscript = False
        if "^" in text or "_" in text:
            buf = ""
            for char in text:
                if char == "^" or char == "_":
                    _append_text(buf, trg)
                    buf = ""
                    sxscript = "sup" if char == "^" else "sub"
                elif sxscript:
                    sx = etree.Element(sxscript)
                    sx.text = char
                    trg.append(sx)
                    sxscript = False
                else:
                    buf += char
            text = buf
        # Append as text
        _append_text(text, trg)
        return sxscript

    def _append_unparsed(self, code, trg):
        pre = etree.Element("span")
        pre.attrib["class"] = "tex-math-unparsed"
        pre.text = f"\\{code.name}{code.args}"
        trg.append(pre)

    def etree_to_html(self, element):
        result = etree.Element("span")
        result.attrib["class"] = "tex-math"
        result.tail = element.tail  # Preserve tail
        self._parse(TexSoup(element.text).expr.all, result)
        return result

    def to_html(self, element):
        """Converts a TeX math expression to HTML markup."""
        if isinstance(element, etree._Element):
            return self.etree_to_html(element)
        elif isinstance(element, str):
            value = self.etree_to_html(etree.fromstring(f"<span>{element}</span>"))
            return etree.tostring(value)
        raise NotImplementedError(f"Cannot convert elements of type {type(element)}")

    def to_unicode(self, element):
        """Converts a TeX math expression to a Unicode string.

        This will perform the same conversions as `to_html()`, but strip out the
        HTML tags afterwards.
        """
        element = self.to_html(element)
        return etree.tostring(element, encoding="unicode", method="text")


# bin/anthology/utils.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import itertools as it
import logging
import os
import re
import requests
import shutil

from lxml import etree
from urllib.parse import urlparse
from xml.sax.saxutils import escape as xml_escape
from typing import Tuple, Optional
from zlib import crc32

from .people import PersonName
from . import data


xml_escape_or_none = lambda t: None if t is None else xml_escape(t)


def is_newstyle_id(anthology_id):
    return anthology_id[0].isdigit()  # New-style IDs are year-first


def is_journal(anthology_id):
    if is_newstyle_id(anthology_id):
        # TODO: this function is sometimes called with "full_id", sometimes with
        # "collection_id", so we're not using `deconstruct_anthology_id` here at
        # the moment
        venue = anthology_id.split("-")[0].split(".")[-1]
        return venue in data.JOURNAL_IDS
    else:
        return anthology_id[0] in ("J", "Q")


def is_volume_id(anthology_id):
    collection_id, volume_id, paper_id = deconstruct_anthology_id(anthology_id)
    return paper_id == "0"


def is_valid_id(id_):
    """
    Determines whether the identifier has a valid Anthology identifier format (paper or volume).
    """
    match = re.match(r"([A-Z]\d{2})-(\d{1,4})", id_)
    if not re.match(r"[A-Z]\d{2}-\d{1,3}", id_):
        return False

    first, rest = match.groups()

    if len(rest) != 4:
        if (
            first.startswith("W")
            or first == "C69"
            or (first == "D19" and int(rest[0]) >= 5)
        ):
            return len(rest) == 2
        else:
            return len(rest) == 1

    return True


def build_anthology_id(
    collection_id: str, volume_id: str, paper_id: Optional[str] = None
) -> str:
    """
    Transforms collection id, volume id, and paper id to a width-padded
    Anthology ID. e.g., ('P18', '1', '1') -> P18-1001.
    """
    if is_newstyle_id(collection_id):
        if paper_id is not None:
            return f"{collection_id}-{volume_id}.{paper_id}"
        else:
            return f"{collection_id}-{volume_id}"
    # pre-2020 IDs
    if (
        collection_id[0] == "W"
        or collection_id == "C69"
        or (collection_id == "D19" and int(volume_id) >= 5)
    ):
        anthology_id = f"{collection_id}-{int(volume_id):02d}"
        if paper_id is not None:
            anthology_id += f"{int(paper_id):02d}"
    else:
        anthology_id = f"{collection_id}-{int(volume_id):01d}"
        if paper_id is not None:
            anthology_id += f"{int(paper_id):03d}"

    return anthology_id


def test_url_code(url):
    """
    Test a URL, returning the result.
    """
    headers = {'user-agent': 'acl-anthology/0.0.1'}
    r = requests.head(url, headers=headers, allow_redirects=True)  # , verify=False)
    return r


def test_url(url):
    """
    Tests a URL, returning True if the URL exists, and False otherwise.
    """
    return test_url_code(url).status_code == requests.codes.ok


def retrieve_url(remote_url: str, local_path: str):
    """
    Saves a URL to a local path. Can handle cookies, e.g., those
    used downloading PDFs from MIT Press (TACL, CL).

    :param remote_url: The URL to download from. Currently supports http only.
    :param local_path: Where to save the file to.
    """
    outdir = os.path.dirname(local_path)
    if outdir != "" and not os.path.exists(outdir):
        os.makedirs(outdir)

    if remote_url.startswith("http"):
        import ssl
        import urllib.request

        cookieProcessor = urllib.request.HTTPCookieProcessor()
        opener = urllib.request.build_opener(cookieProcessor)
        request = urllib.request.Request(
            remote_url,
            headers={
                'User-Agent': 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11'
            },
        )

        with opener.open(request, timeout=1000) as url, open(
            local_path, mode="wb"
        ) as input_file_fh:
            input_file_fh.write(url.read())
    else:
        shutil.copyfile(remote_url, local_path)

    return True


def deconstruct_anthology_id(anthology_id: str) -> Tuple[str, str, str]:
    """
    Transforms an Anthology ID into its constituent collection id, volume id, and paper id
    parts. e.g,

        P18-1007 -> ('P18', '1',  '7')
        W18-6310 -> ('W18', '63', '10')
        D19-1001 -> ('D19', '1',  '1')
        D19-5702 -> ('D19', '57', '2')

    Also can deconstruct Anthology volumes:

        P18-1 -> ('P18', '1', None)
        W18-63 -> ('W18', '63', None)

    For Anthology IDs prior to 2020, the volume ID is the first digit after the hyphen, except
    for the following situations, where it is the first two digits:

    - All collections starting with 'W'
    - The collection "C69"
    - All collections in "D19" where the first digit is >= 5
    """
    collection_id, rest = anthology_id.split("-")
    if is_newstyle_id(anthology_id):
        if "." in rest:
            volume_id, paper_id = rest.split(".")
        else:
            volume_id, paper_id = rest, None
        return (collection_id, volume_id, paper_id)
    # pre-2020 IDs
    if (
        collection_id.startswith("W")
        or collection_id == "C69"
        or (collection_id == "D19" and int(rest[0]) >= 5)
    ):
        if len(rest) == 4:
            return (collection_id, str(int(rest[0:2])), str(int(rest[2:])))
        else:  # Possible Volume only identifier
            return (collection_id, str(int(rest)), None)
    else:
        if len(rest) == 4:
            return (collection_id, str(int(rest[0:1])), str(int(rest[1:])))
        else:  # Possible Volume only identifier
            return (collection_id, str(int(rest)), None)


def get_xml_file(anth_id):
    """
    Returns the XML file containing an Anthology ID.
    """
    collection_id, _, _ = deconstruct_anthology_id(anth_id)
    return os.path.join(
        os.path.dirname(os.path.abspath(__file__)),
        "..",
        "..",
        "data",
        "xml",
        f"{collection_id}.xml",
    )


def get_pdf_dir(anth_id):
    """
    Returns a local path to the directory containing the PDF of the specified Anthology ID.
    """
    collection_id, volume_id, paper_id = deconstruct_anthology_id(anth_id)

    if is_newstyle_id(anth_id):
        venue_name = collection_id.split(".")[1]
        return os.path.join(data.ANTHOLOGY_FILE_DIR, "pdf", venue_name)
    else:
        return os.path.join(
            data.ANTHOLOGY_FILE_DIR, "pdf", collection_id[0], collection_id
        )


def stringify_children(node):
    """Returns the full content of a node, including tags.

    Used for nodes that can have mixed text and HTML elements (like <b> and <i>)."""
    return "".join(
        chunk
        for chunk in it.chain(
            (xml_escape_or_none(node.text),),
            it.chain(
                *(
                    (
                        etree.tostring(child, with_tail=False, encoding=str),
                        xml_escape_or_none(child.tail),
                    )
                    for child in node.getchildren()
                )
            ),
            (xml_escape_or_none(node.tail),),
        )
        if chunk
    ).strip()


def remove_extra_whitespace(text):
    text = text.replace("\n", "").strip()
    # This was profiled to be 2x-4x faster than using re.sub();
    # also cf. https://stackoverflow.com/a/15913564
    while "  " in text:
        text = text.replace("  ", " ")
    return text


def infer_url(filename, template=data.CANONICAL_URL_TEMPLATE):
    """If URL is relative, return the full Anthology URL.
    Returns the canonical URL by default, unless a different
    template is provided."""

    assert (
        "{}" in template or "%s" in template
    ), "template has no substitution text; did you pass a prefix by mistake?"

    if urlparse(filename).netloc:
        return filename
    return template.format(filename)


def infer_attachment_url(filename, parent_id=None):
    if urlparse(filename).netloc:
        return filename
    # Otherwise, treat it as an internal filename
    if parent_id is not None and not filename.startswith(parent_id):
        logging.error(
            f"attachment must begin with paper ID '{parent_id}', but is '{filename}'"
        )
    return infer_url(filename, data.ATTACHMENT_TEMPLATE)


def infer_year(collection_id):
    """Infer the year from the collection ID.

    Many paper entries do not explicitly contain their year.  This function assumes
    that the paper's collection identifier follows the format 'xyy', where x is
    some letter and yy are the last two digits of the year of publication.
    """
    if is_newstyle_id(collection_id):
        return collection_id.split(".")[0]

    assert (
        len(collection_id) == 3
    ), f"Couldn't infer year: unknown volume ID format '{collection_id}' ({type(collection_id)})"
    digits = collection_id[1:]
    if int(digits) >= 60:
        year = f"19{digits}"
    else:
        year = f"20{digits}"

    return year


_MONTH_TO_NUM = {
    "january": 1,
    "february": 2,
    "march": 3,
    "april": 4,
    "may": 5,
    "june": 6,
    "july": 7,
    "august": 8,
    "september": 9,
    "october": 10,
    "november": 11,
    "december": 12,
}


def month_str2num(text):
    """Convert a month string to a number, e.g. February -> 2

    Returns None if the string doesn't correspond to a month.

    Not using Python's datetime here since its behaviour depends on the system
    locale."""
    return _MONTH_TO_NUM.get(text.lower(), None)


class SeverityTracker(logging.Handler):
    def __init__(self, level=logging.NOTSET):
        super().__init__(level=level)
        self.highest = logging.NOTSET

    def emit(self, record):
        if record.levelno > self.highest:
            self.highest = record.levelno


def clean_whitespace(text, strip="left"):
    old_text = text
    if text is not None:
        text = re.sub(r" +", " ", text)
        if strip == "left" or strip == "both":
            text = text.lstrip()
        if strip == "right" or strip == "both":
            text = text.rstrip()
    return text


def indent(elem, level=0, internal=False):
    """
    Enforces canonical indentation: two spaces,
    with each tag on a new line, except that 'author', 'editor',
    'title', and 'booktitle' tags are placed on a single line.

    Adapted from https://stackoverflow.com/a/33956544 .
    """
    # tags that have no internal linebreaks (including children)
    oneline = elem.tag in ("author", "editor", "title", "booktitle", "variant")

    elem.text = clean_whitespace(elem.text)

    if len(elem):  # children
        # Set indent of first child for tags with no text
        if not oneline and (not elem.text or not elem.text.strip()):
            elem.text = "\n" + (level + 1) * "  "

        if not elem.tail or not elem.tail.strip():
            if level:
                elem.tail = "\n" + level * "  "
            else:
                elem.tail = "\n"

        # recurse
        for child in elem:
            indent(child, level + 1, internal=oneline)

        # Clean up the last child
        if oneline:
            child.tail = clean_whitespace(child.tail, strip="right")
        elif not child.tail or not child.tail.strip():
            child.tail = "\n" + level * "  "
    else:
        elem.text = clean_whitespace(elem.text, strip="both")

        if internal:
            elem.tail = clean_whitespace(elem.tail, strip="none")
        elif not elem.tail or not elem.tail.strip():
            elem.tail = "\n" + level * "  "


def parse_element(xml_element):
    attrib = {}
    if xml_element is None:
        return attrib

    for element in xml_element:
        # parse value
        tag = element.tag.lower()
        if tag in ("abstract", "title", "booktitle"):
            tag = f"xml_{tag}"
            value = element
        elif tag == "url":
            tag = "xml_url"
            value = element.text
        elif tag == "attachment":
            value = {
                "filename": element.text,
                "type": element.get("type", "attachment"),
                "url": element.text,
            }
        elif tag in ("author", "editor"):
            id_ = element.attrib.get("id", None)
            value = (PersonName.from_element(element), id_)
        elif tag == "erratum":
            value = {"value": element.text, "id": element.get("id"), "url": element.text}
        elif tag == "revision":
            value = {
                "value": element.get("href"),
                "id": element.get("id"),
                "url": element.get("href"),
                "explanation": element.text,
            }
        elif tag == "mrf":
            value = {"filename": element.text, "src": element.get("src")}
        elif tag == "video":
            tag = "video"
            # Skip videos where permission was not granted (these should be marked as private anyway in Vimeo)
            if element.get("permission", "true") == "false":
                continue
            value = element.get("href")
        elif tag in ("dataset", "software"):
            value = {"filename": element.text, "type": tag, "url": element.text}
            tag = "attachment"
        elif tag == "pwccode":
            value = {
                "url": element.get("url"),
                "additional": element.get("additional"),
                "name": element.text,
            }
        elif tag == "pwcdataset":
            value = {"url": element.get("url"), "name": element.text}
        else:
            value = element.text

        if tag in data.LIST_ELEMENTS:
            try:
                attrib[tag].append(value)
            except KeyError:
                attrib[tag] = [value]
        else:
            attrib[tag] = value

    return attrib


def make_simple_element(tag, text=None, attrib=None, parent=None, namespaces=None):
    """Convenience function to create an LXML node"""
    el = (
        etree.Element(tag, nsmap=namespaces)
        if parent is None
        else etree.SubElement(parent, tag)
    )
    if text:
        el.text = text
    if attrib:
        for key, value in attrib.items():
            el.attrib[key] = value
    return el


def compute_hash(value: bytes) -> str:
    checksum = crc32(value) & 0xFFFFFFFF
    return f"{checksum:08x}"


def compute_hash_from_file(path: str) -> str:
    with open(path, "rb") as f:
        return compute_hash(f.read())


# bin/anthology/venues.py

# -*- coding: utf-8 -*-
#
# Copyright 2019-2020 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict
from copy import deepcopy
from slugify import slugify
import logging as log
import re
import yaml

try:
    from yaml import CLoader as Loader
except ImportError:
    from yaml import Loader

from .utils import is_newstyle_id, build_anthology_id, deconstruct_anthology_id
from anthology.data import VENUE_FORMAT


class VenueIndex:
    def __init__(self, srcdir=None):
        self.venues, self.letters, self.joint_map = {}, {}, defaultdict(list)
        self.acronyms = {}  # acronym -> venue
        self.acronyms_by_key = {}  # slug -> acronym
        self.venue_dict = None
        if srcdir is not None:
            self.load_from_dir(srcdir)

    @staticmethod
    def get_slug(acronym):
        """The acronym can contain a hyphen, whereas the slug must match VENUE_FORMAT."""
        slug = slugify(acronym.replace("-", ""))
        assert (
            re.match(VENUE_FORMAT, slug) is not None
        ), f"Proposed slug '{slug}' of venue '{acronym}' doesn't match {VENUE_FORMAT}"
        return slug

    def add_venue(self, acronym, title, is_acl=False, url=None):
        """
        Adds a new venue.
        """
        slug = VenueIndex.get_slug(acronym)

        self.venue_dict[slug] = {"acronym": acronym, "name": title}
        if is_acl:
            self.venue_dict[slug]["is_acl"] = True
        if url is not None:
            self.venue_dict[slug]["url"] = url

    def dump(self, directory):
        """
        Dumps the venue database to file.
        """
        with open(f"{directory}/yaml/venues.yaml", "wt") as f:
            print(yaml.dump(self.venue_dict, allow_unicode=True), file=f)

    def load_from_dir(self, directory):
        with open(f"{directory}/yaml/venues.yaml", "r") as f:
            self.venue_dict = yaml.load(f, Loader=Loader)
            venue_dict = deepcopy(self.venue_dict)
            for key, val in venue_dict.items():
                if "acronym" not in val:
                    log.critical(f"Venues must have 'acronym' - none defined for '{key}'")
                if "name" not in val:
                    log.error(f"Venues must have 'name' - none defined for '{key}'")
                if val["acronym"] in self.venues:
                    log.critical(
                        f"Venue acronyms must be unique - '{val['acronym']}' used"
                        f" for '{self.venues[val['acronym']]['slug']}' and '{key}'"
                    )

                if "is_toplevel" not in val:  # defaults to False
                    val["is_toplevel"] = False
                if "is_acl" not in val:  # defaults to False
                    val["is_acl"] = False
                if "joint" in val:
                    if isinstance(val["joint"], str):
                        val["joint"] = [val["joint"]]
                val["years"] = set()
                val["volumes"] = list()
                val["slug"] = key
                # for legacy reasons, this dict is still indexed by acronym
                # rather than key (that's what I get for not using proper
                # encapsulation --MB)
                self.venues[val["acronym"]] = val
                self.acronyms_by_key[key] = val["acronym"]
                self.acronyms[val["acronym"]] = val

                if "oldstyle_letter" in val:
                    if not val["is_toplevel"]:
                        log.error(
                            f"Venues with old-style letter must be top-level - '{key}' is not"
                        )
                    self.letters[val["oldstyle_letter"]] = val["acronym"]

        with open(f"{directory}/yaml/joint.yaml", "r") as f:
            map_dict = yaml.load(f, Loader=Loader)
            for venue, data in map_dict.items():
                acronym = self.acronyms_by_key[venue]
                if isinstance(data, dict):
                    idlist = [id_ for ids in data.values() for id_ in ids]
                elif isinstance(data, list):
                    idlist = data
                else:
                    log.exception(
                        f"Values in joint.yaml must be dict or list, found: {type(data)}"
                    )
                for id_ in idlist:
                    self.joint_map[id_].append(acronym)

    def get_by_letter(self, letter):
        """Get a venue acronym by first letter (e.g., Q -> TACL)."""
        return self.letters.get(letter, None)

    def get_by_acronym(self, acronym):
        """Get a venue object by its acronym (assumes acronyms are unique)."""
        try:
            return self.acronyms[acronym]
        except KeyError:
            log.critical(f"Unknown venue acronym: {acronym}")

    def get_main_venue(self, anthology_id):
        """Get a venue acronym by anthology ID (e.g., acl -> ACL)."""
        collection_id, volume_id, _ = deconstruct_anthology_id(anthology_id)
        if is_newstyle_id(collection_id):
            return self.acronyms_by_key[collection_id.split(".")[-1]]
        else:  # old-style ID
            main_venue = self.get_by_letter(collection_id[0])
            if main_venue is None:
                # If there's no association with the letter, use joint.yaml to
                # get the venue.  As of 06/2021 this is only used for "O"
                # (ROCLING/IJCLCLP).
                try:
                    main_venue = self.joint_map[
                        build_anthology_id(collection_id, volume_id, None)
                    ][0]
                except (KeyError, IndexError):
                    log.critical(f"Old-style ID {anthology_id} isn't assigned any venue!")
            return main_venue

    def get_associated_venues(self, anthology_id):
        """Get a list of all venue acronyms for a given (volume) anthology ID."""
        main_venue = self.get_main_venue(anthology_id)
        venues = [main_venue]
        if "joint" in self.venues[main_venue]:
            venues += self.venues[main_venue]["joint"]
        if anthology_id in self.joint_map:
            venues += self.joint_map[anthology_id]
        return sorted(set(venues))

    def register(self, volume):
        """Register a proceedings volume with all associated venues."""
        venues = self.get_associated_venues(volume.full_id)
        for venue in venues:
            self.venues[venue]["volumes"].append(volume.full_id)
            self.venues[venue]["years"].add(volume.get("year"))
        return venues

    def items(self):
        return self.venues.items()


# bin/anthology/volumes.py

# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import cached_property
import re
import logging as log

from . import data
from .papers import Paper
from .venues import VenueIndex
from .sigs import SIGIndex
from .utils import (
    build_anthology_id,
    parse_element,
    is_journal,
    month_str2num,
    infer_url,
    infer_year,
)


class Volume:
    def __init__(
        self,
        collection_id,
        volume_id,
        ingest_date,
        meta_data,
        venue_index: VenueIndex,
        sig_index: SIGIndex,
        formatter,
    ):
        """Instantiate a proceedings volume.

        `venue_index` and `sig_index` are used to find venues and SIGs
        associated with this proceedings volume.
        """
        self.collection_id = collection_id
        self._id = volume_id
        self.ingest_date = ingest_date
        self.formatter = formatter
        self.venue_index = venue_index
        self._set_meta_info(meta_data)
        self.attrib["venues"] = venue_index.get_associated_venues(self.full_id)
        self.attrib["sigs"] = sig_index.get_associated_sigs(self.full_id)
        self.content = []
        self.has_abstracts = False
        self.has_frontmatter = False

    @staticmethod
    def from_xml(
        volume_xml, collection_id, venue_index: VenueIndex, sig_index: SIGIndex, formatter
    ):

        volume_id = volume_xml.attrib["id"]
        # The date of publication, defaulting to earlier than anything we'll encounter
        ingest_date = volume_xml.attrib.get("ingest-date", data.UNKNOWN_INGEST_DATE)
        meta_data = parse_element(volume_xml.find("meta"))
        # Though metadata uses "booktitle", switch to "title" for compatibility with downstream scripts
        meta_data["title"] = formatter(meta_data["xml_booktitle"], "plain")

        volume = Volume(
            collection_id,
            volume_id,
            ingest_date,
            meta_data,
            venue_index,
            sig_index,
            formatter,
        )

        front_matter_xml = volume_xml.find("frontmatter")
        if front_matter_xml is not None:
            front_matter = Paper.from_xml(
                front_matter_xml, volume, formatter, venue_index
            )
            volume.add_frontmatter(front_matter)

        return volume

    @cached_property
    def url(self):
        # If <url> field not present, use ID.
        # But see https://github.com/acl-org/acl-anthology/issues/997.
        return infer_url(self.attrib.get("xml_url", self.full_id))

    @cached_property
    def pdf(self):
        url = self.attrib.get("xml_url", None)
        if url is not None:
            return infer_url(url, template=data.PDF_LOCATION_TEMPLATE)
        return None

    def _set_meta_info(self, meta_data):
        """Derive journal title, volume, and issue no. used in metadata.

        This function replicates functionality that was previously hardcoded in
        'app/helpers/papers_helper.rb' of the Rails app."""
        self.attrib = meta_data
        if "author" in self.attrib:
            # Authors of the front matter are the volume's editors
            self.attrib["editor"] = self.attrib["author"]
            del self.attrib["author"]

        # Some volumes don't set this---but they should!
        if "year" not in self.attrib:
            self.attrib["year"] = infer_year(self.collection_id)

        self.attrib["meta_date"] = self.get("year")
        if "month" in self.attrib:
            month = month_str2num(self.get("month"))
            if month is not None:
                self.attrib["meta_date"] = f"{self.get('year')}/{month}"
        if is_journal(self.collection_id):
            self.attrib["meta_journal_title"] = data.get_journal_title(
                self.collection_id, self.attrib["title"]
            )
            volume_no = re.search(
                r"Volume\s*(\d+)", self.attrib["title"], flags=re.IGNORECASE
            )
            if volume_no is not None:
                self.attrib["meta_volume"] = volume_no.group(1)
            issue_no = re.search(
                r"(Number|Issue)\s*(\d+-?\d*)", self.attrib["title"], flags=re.IGNORECASE
            )
            if issue_no is not None:
                self.attrib["meta_issue"] = issue_no.group(2)

    @property
    def volume_id(self):
        return self._id

    @cached_property
    def full_id(self):
        return build_anthology_id(self.collection_id, self.volume_id)

    @property
    def paper_ids(self):
        return [paper.full_id for paper in self.content]

    def add_frontmatter(self, frontmatter):
        self.has_frontmatter = True
        self.append(frontmatter)

    def append(self, paper):
        self.content.append(paper)
        if paper.has_abstract:
            self.has_abstracts = True

    def get(self, name, default=None):
        try:
            return self.attrib[name]
        except KeyError:
            return default

    def get_title(self, form="xml"):
        """Returns the paper title, optionally formatting it.

        Accepted formats:
          - xml:   Include any contained XML tags unchanged
          - plain: Strip all XML tags, returning only plain text
          - html:  Convert XML tags into valid HTML tags
        """
        return self.formatter(self.get("xml_booktitle"), form)

    def as_dict(self):
        value = self.attrib.copy()
        value["url"] = self.url
        if self.pdf:
            value["pdf"] = self.pdf
        return value

    def __len__(self):
        return len(self.content)

    def __iter__(self):
        return self.content.__iter__()


# bin/fixedcase/common.py

#!/usr/bin/env python3

import re, sys, os
import inspect
import nltk.tokenize, nltk.corpus
from collections import defaultdict

from nltk.tokenize.treebank import TreebankWordDetokenizer


def is_hyphen(s):
    return s in ("-", "–")


def no_hyphens(ws):
    return tuple(w for w in ws if not is_hyphen(w))


def get_text(node):
    result = []

    def visit(node):
        if node is not None:
            if node.text is not None:
                result.append(node.text)
            for child in node:
                visit(child)
            if node.tail is not None:
                result.append(node.tail)

    visit(node)
    return "".join(result)


def tokenize(s):
    """Splits tokens (hyphens/slashes count as separate tokens)."""
    tokens = []
    # NLTK tokenizer uses PTB standard, which doesn't split on hyphens or slashes
    for tok in nltk.tokenize.word_tokenize(s):
        # tokenizer normalizes quotes etc., so we need to detokenize later
        tokens.extend([t for t in re.split(r"([-–/])", tok) if t != ""])
    return tokens


def fixedcase_word(w, truelist=None):
    """Returns True if w should be fixed-case, None if unsure."""
    if truelist is not None and w in truelist:
        return True
    if any(c.isupper() for c in w[1:]):
        # tokenized word with noninitial uppercase
        return True
    if len(w) == 1 and w.isupper() and w not in {'A', 'K', 'N'}:
        # single uppercase letter
        return True
    if len(w) == 2 and w[1] == '.' and w[0].isupper():
        # initial with period
        return True


def fixedcase_prefix(ws, truelist=None, phrase_truelist=None):
    """Returns a list of 1 or more bools: True if some prefix of the tuple 'ws' should be fixed-case,
    False if not, None if unsure."""
    # phrase_truelist is sorted in descending order by phrase length
    if phrase_truelist is not None:
        for n, truelist_bin in phrase_truelist:
            if ws[:n] in truelist_bin:
                return [True] * n
            if len(no_hyphens(ws)) >= n and no_hyphens(ws)[:n] in truelist_bin:
                # no hyphens in truelist entries
                bs = []
                i = 0
                for tok in ws:
                    if is_hyphen(tok):
                        bs.append(False)
                    else:
                        bs.append(True)
                        i += 1
                        if i == n:
                            break
                return bs
    if ws[0] in {'L', 'D'} and len(ws) >= 2 and ws[1] == '’':
        # French contractions: don't apply fixed-case
        return [False, False]
    return [fixedcase_word(ws[0], truelist=truelist)]


def fixedcase_title(
    ws, truelist=None, phrase_truelist=None, amodifiers=None, ndescriptors=None
):
    """Returns a list of bools: True if w should be fixed-case, False if
    not, None if unsure."""

    bs = []
    ws = tuple(ws)
    i = 0
    while i < len(ws):
        b = fixedcase_prefix(ws[i:], truelist=truelist, phrase_truelist=phrase_truelist)
        if i == 0:
            pass
        elif b[0] and amodifiers and ws[i - 1] in amodifiers:  # e.g. North America
            bs[-1] = True
        elif b[0] and is_hyphen(ws[i - 1]) and amodifiers and ws[i - 2] in amodifiers:
            bs[-2] = True
        elif not b[0] and bs[-1] and ndescriptors and ws[i] in ndescriptors:
            # "<name> <ndescriptor>", e.g. Columbia University
            b[0] = True
        elif ndescriptors and i >= 2 and ws[i - 1] == "of" and ws[i - 2] in ndescriptors:
            # "<ndescriptor> of <name>", e.g. University of Edinburgh
            if b[0]:
                bs[-2] = True
            else:
                print(ws[i - 2 :], file=sys.stderr)
                # mainly: University of X where X is not in the truelist
        bs.extend(b)
        i += len(b)
    return bs


def replace_node(old, new):
    old.clear()
    old.tag = new.tag
    old.attrib.update(new.attrib)
    old.text = new.text
    old.extend(new)
    old.tail = new.tail


def append_text(node, text):
    if len(node) == 0:
        if node.text is None:
            node.text = ""
        node.text += text
    else:
        if node[-1].tail is None:
            node[-1].tail = ""
        node[-1].tail += text


def load_lists():
    truelist = set()
    phrase_truelist = defaultdict(set)
    module_file = inspect.getfile(inspect.currentframe())
    module_dir = os.path.dirname(os.path.abspath(module_file))
    truelist_file = os.path.join(module_dir, "truelist")
    for line in open(truelist_file):
        line = line.split("#")[0].strip()
        if line == "":
            continue
        assert not any(
            is_hyphen(c) for c in line
        ), f'Truelist entries should not contain hyphens: {line}'
        if ' ' not in line:
            truelist.add(line)
        else:
            toks = tuple(tokenize(line))
            phrase_truelist[len(toks)].add(toks)  # group phrases by number of tokens
    phrase_truelist = sorted(
        phrase_truelist.items(), reverse=True
    )  # bins sorted by phrase length
    special_file = os.path.join(module_dir, "special-case-titles")
    with open(special_file) as inF:
        special_titles = {
            line.strip().lower(): line.strip() for line in inF if line.strip()
        }
    amodifiers = (
        'North',
        'South',
        'East',
        'West',
        'Northeast',
        'Northwest',
        'Southeast',
        'Southwest',
        'Central',
        'Northern',
        'Southern',
        'Eastern',
        'Western',
        'Northeastern',
        'Northwestern',
        'Southeastern',
        'Southwestern',
        'Modern',
        'Ancient',
    )  # use subsequent word to determine fixed-case. will miss hyphenated modifiers (e.g. South-East)
    ndescriptors = (
        'Bay',
        'Coast',
        'Gulf',
        'Island',
        'Isle',
        'Lake',
        'Republic',
        'University',
    )  # use preceding word to determine fixed-case

    return truelist, phrase_truelist, special_titles, amodifiers, ndescriptors


# bin/fixedcase/protect.py

#!/usr/bin/env python3

# protect.py <infile> <outfile>
# looks for file "truelist" in current dir

# cd data/xml
# for i in *xml ; do (cd ../../tools/fixedcase/ ; python3 ./protect.py ../../data/xml/$i /tmp/$i ; echo $i ); done > log


import lxml.etree as ET
import os
import sys
import copy
import itertools
import inspect

from collections import defaultdict

if __name__ == "__main__":
    from common import *
else:
    from .common import *

# recursive helper called by protect
# protect text of "node", including children, and tails of children
def protect_recurse(node, recased):
    if node.tag in ("fixed-case", "tex-math"):  # already protected text, or math
        newnode = copy.deepcopy(node)  # don't need to modify descendents
        newnode.tail = None  # tail will be protected by caller
        return newnode
    newnode = ET.Element(node.tag, node.attrib)

    def process(text, rc):
        i = 0
        for upper, chars in itertools.groupby(rc[: len(text)], lambda c: c.isupper()):
            charstr = "".join(chars)
            if upper:
                p = ET.Element("fixed-case")
                p.text = charstr
                newnode.append(p)
            else:
                append_text(newnode, text[i : i + len(charstr)])

            assert text[i : i + len(charstr)].lower() == charstr.lower(), (
                i,
                text,
                charstr,
            )
            i += len(charstr)

    if node.text:
        process(node.text, recased)
        recased = recased[len(node.text) :]
    for child in node:
        protected_child = protect_recurse(child, recased)
        recased = recased[len(get_text(protected_child)) :]
        newnode.append(protected_child)
        if child.tail:
            process(child.tail, recased)
            recased = recased[len(child.tail) :]

    return newnode


def protect(node):
    rawtext = get_text(node).strip()
    recased = None
    if rawtext.lower() in special_titles:
        recased = special_titles[rawtext.lower()]
    else:
        text = tokenize(rawtext)
        fixed = fixedcase_title(
            text,
            truelist=truelist,
            phrase_truelist=phrase_truelist,
            amodifiers=amodifiers,
            ndescriptors=ndescriptors,
        )
        if any(fixed):
            # Generate the recased string so we know where to look in the XML
            # to apply fixed-case
            recasedtoks = [(w if b else w.lower()) for w, b in zip(text, fixed)]
            recased = TreebankWordDetokenizer().detokenize(recasedtoks)
            # PTB (de)tokenizer doesn't think of hyphens as separate tokens,
            # so we need to manually detokenize them.
            # Assuming the only edits that need to be made are adding/deleting
            # spaces, the following will work:
            i = 0
            while i < len(rawtext):
                # scan rawtext from left to right and adjust recased by adding/removing
                # spaces until it matches
                t = rawtext[i]
                assert i < len(recased), ((i, t), rawtext, recased)
                c = recased[i]
                if t.isspace() and not c.isspace():  # may be ' ' or '\n'
                    # add space to recased
                    recased = recased[:i] + t + recased[i:]
                    i += 1
                elif c.isspace() and not t.isspace():
                    # remove space from recased
                    recased = recased[:i] + recased[i + 1 :]
                    # don't increment i
                elif t != c and t.isspace() and c.isspace():
                    recased = recased[:i] + t + recased[i + 1 :]
                    i += 1
                else:
                    assert t == c or t.lower() == c.lower(), (
                        (i, t, c),
                        rawtext,
                        recased,
                        text,
                    )
                    i += 1
            if len(recased) > len(rawtext):
                recased = recased[: len(rawtext)]
            assert rawtext.lower() == recased.lower(), (rawtext, recased)

    if recased:
        newnode = protect_recurse(node, recased)
        newnode.tail = node.tail  # tail of top level is not protected
        replace_node(node, newnode)


# Read in the truelist (list of words that should always be protected)
truelist, phrase_truelist, special_titles, amodifiers, ndescriptors = load_lists()

if __name__ == "__main__":
    infile, outfile = sys.argv[1:]

    tree = ET.parse(infile)
    if not tree.getroot().tail:
        tree.getroot().tail = "\n"
    for paper in tree.getroot().findall(".//paper"):
        for title in paper.xpath("./title|./booktitle"):
            protect(title)
    tree.write(outfile, encoding="UTF-8", xml_declaration=True)


# bin/fixedcase/protect_allcaps.py

#!/usr/bin/env python3

# protect_allcaps.py <infile> <outfile>
# looks for file "truelist" in current dir

# Simpler version of protect.py for titles written in all caps.

import lxml.etree as ET
import os
import sys
import copy
import itertools
import inspect


if __name__ == "__main__":
    from common import *
else:
    from .common import *


def protect(node):
    if node.tag == "fixed-case":  # already protected
        newnode = copy.deepcopy(node)  # don't need to modify descendents
        newnode.tail = None  # tail will be protected by caller
        return newnode
    newnode = ET.Element(node.tag, node.attrib)

    def process(text):
        if text is None:
            return
        tokens = list(re.split(r'([\s:.,?-])', text))
        for token in tokens:
            if token == token.upper() and token in truelist:
                print(token, file=sys.stderr)
                for upper, chars in itertools.groupby(truelist[token], str.isupper):
                    if upper:
                        p = ET.Element("fixed-case")
                        p.text = "".join(chars).upper()
                        newnode.append(p)
                    else:
                        append_text(newnode, "".join(chars).upper())
            else:
                append_text(newnode, token)

    process(node.text)
    for child in node:
        newnode.append(protect(child))
        process(child.tail)
    return newnode


# Read in the truelist (list of words that should always be protected)
truelist = {}
module_file = inspect.getfile(inspect.currentframe())
module_dir = os.path.dirname(os.path.abspath(module_file))
truelist_file = os.path.join(module_dir, "truelist")
for line in open(truelist_file):
    line = line.split("#")[0].strip()
    if line == "":
        continue
    truelist[line.upper()] = line

# The truelist does not contain any acronyms, because normally they
# are marked as fixed-case by virtue of being written in all caps. But
# here we need an explicit list.

for word in "ACL ATIS ATN BBN CCG CMU COLING GPSG MT NIST NLP SRI TIPSTER TREC".split():
    truelist[word.upper()] = word

if __name__ == "__main__":
    infile, outfile = sys.argv[1:]

    tree = ET.parse(infile)
    if not tree.getroot().tail:
        tree.getroot().tail = "\n"

    for paper in tree.getroot().xpath(".//paper"):
        for title in paper.xpath("./title|./booktitle"):
            newtitle = protect(title)
            newtitle.tail = title.tail
            replace_node(title, newtitle)
    tree.write(outfile, encoding="UTF-8", xml_declaration=True)


# bin/fixedcase/shrink.py

#!/usr/bin/env python3

import lxml.etree as etree
import sys
import itertools
import copy
from common import *

tree = etree.parse(sys.argv[1])
if not tree.getroot().tail:
    tree.getroot().tail = "\n"

for paper in tree.getroot().findall("paper"):
    for old in paper.xpath("./title|./booktitle"):
        new = etree.Element(old.tag)
        if old.text:
            append_text(new, old.text)
        for child in old:
            if child.tag == "fixed-case":
                assert len(child) == 0
                for upper, chars in itertools.groupby(child.text, lambda c: c.isupper()):
                    if upper:
                        newchild = etree.Element("fixed-case")
                        newchild.text = "".join(chars)
                        new.append(newchild)
                    else:
                        append_text(new, "".join(chars))
                if child.tail:
                    append_text(new, child.tail)
            else:
                new.append(copy.deepcopy(child))
        new.tail = old.tail
        replace_node(old, new)

tree.write(sys.argv[2], xml_declaration=True, encoding="UTF-8")


# bin/fixedcase/train.py

#!/usr/bin/env python3

import xml.etree.ElementTree as ET
import collections
import sys
from common import *

falselist = set([w for w in nltk.corpus.words.words() if w.islower()])
# Note: this is only a list of lemmas, so words like "frames" don't appear

if __name__ == "__main__":
    c = collections.defaultdict(collections.Counter)
    for xmlfile in sys.argv[1:]:
        tree = ET.parse(xmlfile)
        for paper in tree.getroot().findall(".//paper"):
            for abstract in paper.findall("abstract"):
                for w in tokenize(get_text(abstract)):
                    c[w.lower()][w] += 1
    truelist = []
    for w_lower in c:
        if w_lower in falselist:
            continue
        if w_lower != max(c[w_lower], key=c[w_lower].get):
            for w in c[w_lower]:
                if fixedcase_word(w) == None:
                    truelist.append(w)
    truelist.sort()

    with open("truelist-auto", "w") as outfile:
        print("# Automatically generated by train.py", file=outfile)
        for w in truelist:
            print(w, file=outfile)


# bin/fixedcase/train_phrasal.py

#!/usr/bin/env python3

import xml.etree.ElementTree as ET
import sys
from collections import Counter

from common import *

if __name__ == "__main__":
    c = Counter()
    # First pass: count 2-, 3-, 4-, and 5-grams (counting hyphens as spaces)
    # where at least the first and last words have initial caps
    for xmlfile in sys.argv[1:]:
        print(xmlfile)
        tree = ET.parse(xmlfile)
        for paper in tree.getroot().findall(".//paper"):
            for abstract in paper.findall("abstract"):
                toks = tokenize(get_text(abstract).replace('-', ' '))
                for i, w in enumerate(toks[:-1]):
                    if w[0].isupper():
                        queue = []
                        if toks[i + 1][0].isupper():
                            queue.append(tuple(toks[i : i + 2]))  # bigram
                        if i + 2 < len(toks) and toks[i + 2][0].isupper():
                            queue.append(tuple(toks[i : i + 3]))  # trigram
                        if i + 3 < len(toks) and toks[i + 3][0].isupper():
                            queue.append(tuple(toks[i : i + 4]))  # 4-gram
                        if i + 4 < len(toks) and toks[i + 4][0].isupper():
                            queue.append(tuple(toks[i : i + 5]))  # 5-gram
                        for phr in queue:
                            if '.' in phr or ',' in phr or '(' in phr or ')' in phr:
                                continue
                            c[phr] += 1

    print(f'{len(c)} entries before filtering')
    for k, v in c.copy().items():
        if v < 3:
            del c[k]
    print(f'{len(c)} entries after min 3 threshold')

    d = {' '.join(k).lower(): Counter() for k in sorted(c)}
    print(f'{len(d)} unique entries ignoring case')

    # Second pass: filter to n-grams that are capitalized in most abstracts,
    # and that appear in titles

    # Count each way of capitalizing the phrase (first match per abstract),
    # and occurrences in titles regardless of capitalization
    intitles = Counter()
    for xmlfile in sys.argv[1:]:
        print(xmlfile)
        tree = ET.parse(xmlfile)
        for paper in tree.getroot().findall(".//paper"):
            for abstract in paper.findall("abstract"):
                toks = tokenize(get_text(abstract).replace('-', ' '))
                s = ' '.join(toks)
                s_lower = s.lower()
                for q in d:
                    i = s_lower.find(q)
                    if i > -1:
                        d[q][s[i : i + len(q)]] += 1
            for title in paper.findall("title") + paper.findall("booktitle"):
                titleS = ' '.join(tokenize(get_text(title).replace('-', ' ').lower()))
                for q in d:
                    if f' {q} ' in f' {titleS} ':
                        intitles[q] += 1

    print(f'{len(intitles)} of the entries occur in titles')

    # # normalize hyphens
    # for q in d.copy():
    #     if ' - ' in q:
    #         hyphC = d[q]
    #         dehyphC = Counter({k.replace(' - ', ' '): v for k,v in hyphC.items()})
    #         d.setdefault(q.replace(' - ', ' '), Counter())
    #         d[q.replace(' - ', ' ')] += dehyphC
    #         del d[q]
    #         intitles[q.replace(' - ', ' ')] += intitles[q]
    #         del intitles[q]
    # print(f'{len(d)} unique entries after normalizing hyphens')

    old_uni_truelist, old_phrase_truelist, amodifiers, ndescriptors = load_lists()

    newC = Counter()
    filterLater = set()
    for q in d:
        if (
            intitles[q] > 0 and d[q][q] / sum(d[q].values()) <= 0.25
        ):  # no more than 25% instances of the phrase are all-lowercase
            top_spelling = d[q].most_common(1)[0][0]
            # this is a spelling where the first and last word are capitalized
            if (
                top_spelling[0].isupper()
                and top_spelling[top_spelling.rindex(' ') + 1].isupper()
            ):
                # store the most frequent capitalization
                # with # of times the phrase occurred non-lowercased
                newC[top_spelling] = sum(d[q].values()) - d[q][q]

                # would this be truecased already?
                toks = top_spelling.split()
                # all-caps title heuristic can be unfair on short n-grams,
                # so append some dummy words
                bs = fixedcase_title(
                    toks + ['The', 'the', 'The', 'the', 'The'],
                    truelist=old_uni_truelist,
                    phrase_truelist=old_phrase_truelist,
                    amodifiers=amodifiers,
                    ndescriptors=ndescriptors,
                )

                # does this spelling meet all the criteria for a truelist phrase?
                # if not, count it for now (to apply inclusion-exclusion)
                # but mark it for removal later
                keep = False
                for tok, b in zip(toks, bs):
                    if (
                        not b
                        and tok[0].isupper()
                        and not any(c.isupper() for c in tok[1:])
                    ):
                        keep = True
                        break
                if not keep:
                    filterLater.add(top_spelling)

    print(f'{len(newC)} usually capitalized phrases that occur in titles')

    # print(newC)
    newC1 = newC.copy()

    # For 3-, 4-, and 5-grams, subtract counts for subsumed (n-1)-grams
    for phr, n in sorted(newC.copy().items(), key=lambda s: s.count(' '), reverse=True):
        if phr.count(' ') > 1:
            toks = tuple(phr.split())
            newC[' '.join(toks[:-1])] -= n
            newC[' '.join(toks[1:])] -= n
            if len(toks) > 3:  # inclusion-exclusion
                # correct for double-counting of overlap between 2 spans just decremented
                newC[' '.join(toks[1:-1])] += n

                if len(toks) > 4:
                    newC[' '.join(toks[2:-1])] -= n
                    newC[' '.join(toks[1:-2])] -= n

    # print(newC)

    new_truelist = [phr for phr in newC1 if newC[phr] >= 3 and phr not in filterLater]
    new_truelist.sort()

    # truelist = []
    # for w_lower in c:
    #     if w_lower in falselist:
    #         continue
    #     if w_lower != max(c[w_lower], key=c[w_lower].get):
    #         for w in c[w_lower]:
    #             if fixedcase_word(w) == None:
    #                 truelist.append(w)
    # truelist.sort()

    with open("truelist-phrasal-auto", "w") as outfile:
        print("# Automatically generated by train_phrasal.py", file=outfile)
        for phr in new_truelist:
            print(phr, file=outfile)


# bin/add_attachment.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2021 Xinru Yan <xinru1414@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Used to add attachment revision as an additional attachament to the Anthology.

This script is heavily adopted from add_attachments.py
Only adding attachment without reading in .csv file and keeping a log

Usage:

    add_attachment.py anth_id attachment_revision_URL 'ATTACHMENT_TYPE'
"""

import argparse
import filetype
import os
import shutil
import ssl
import sys
import tempfile

from anthology.utils import (
    build_anthology_id,
    deconstruct_anthology_id,
    indent,
    compute_hash,
)

import lxml.etree as ET
import urllib.request
from datetime import datetime

ALLOWED_TYPES = ["pdf", "pptx", "zip"]
ATTACHMENT_TYPES = "Poster Presentation Note Software Supplementary Dataset".split()


def add_attachment(anthology_id, path, attach_type, overwrite=False):
    """
    Adds a single attachment to the Anthology data files.

    Arguments:
    - The ACL ID of the paper (e.g., P17-1012)
    - The path to the attachment (can be a URL)
    - The attachment type (poster, presentation, note, software)
    - Whether to overwrite the downloaded file.
    """

    collection_id, volume_id, paper_id = deconstruct_anthology_id(anthology_id)

    if path.startswith("http"):
        _, input_file_path = tempfile.mkstemp()
        try:
            print(
                f"-> Downloading file from {path} to {input_file_path}", file=sys.stderr
            )
            request = urllib.request.Request(path, headers={'User-Agent': 'Mozilla/5.0'})
            with urllib.request.urlopen(request) as url, open(
                input_file_path, mode="wb"
            ) as input_file_fh:
                input_file_fh.write(url.read())
        except ssl.SSLError:
            raise Exception(f"Could not download {path}")
        except Exception as e:
            raise e
    else:
        input_file_path = path

    file_extension = path.replace("?dl=1", "").split(".")[-1]
    # Many links from file sharing services are not informative and don't have
    # extensions, so we could try to guess.
    if file_extension not in ALLOWED_TYPES:
        detected = filetype.guess(input_file_path)
        if detected is not None:
            file_extension = detected.mime.split("/")[-1]
            if file_extension not in ALLOWED_TYPES:
                print(
                    f"Could not determine file extension for {anthology_id} at {path}",
                    file=sys.stderr,
                )

    with open(input_file_path, "rb") as f:
        checksum = compute_hash(f.read())

    # Update XML
    xml_file = os.path.join(
        os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
    )
    tree = ET.parse(xml_file)

    attachment_file_name = f"{anthology_id}.{attach_type}.{file_extension}"

    paper = tree.getroot().find(f"./volume[@id='{volume_id}']/paper[@id='{paper_id}']")
    if paper is not None:
        # Check if attachment already exists
        for attachment in paper.findall("attachment"):
            if attachment.text == attachment_file_name:
                print(
                    f"-> attachment {attachment_file_name} already exists in the XML",
                    file=sys.stderr,
                )
                break
        else:
            attachment = ET.Element("attachment")
            attachment.attrib["type"] = attach_type.lower()
            attachment.attrib["hash"] = checksum
            attachment.text = attachment_file_name

            paper.append(attachment)
            indent(tree.getroot())
            tree.write(xml_file, encoding="UTF-8", xml_declaration=True)
            print(
                f"-> added attachment {attachment_file_name} to the XML", file=sys.stderr
            )

    else:
        print(f"Paper {anthology_id} not found in the Anthology", file=sys.stderr)

    # Make sure directory exists
    output_dir = os.path.join(args.attachment_root, collection_id[0], collection_id)
    if not os.path.exists(output_dir):
        #        print(f"-> Creating directory {output_dir}", file=sys.stderr)
        os.makedirs(output_dir)

    # Copy file
    dest_path = os.path.join(output_dir, attachment_file_name)
    if os.path.exists(dest_path) and not overwrite:
        print(
            f"-> target file {dest_path} already in place, refusing to overwrite",
            file=sys.stderr,
        )
        return None

    shutil.copy(input_file_path, dest_path)
    os.chmod(dest_path, 0o644)
    print(f"-> copied {input_file_path} to {dest_path} and fixed perms", file=sys.stderr)

    # Clean up
    if path.startswith("http"):
        os.remove(input_file_path)

    return dest_path


def main(args):
    add_attachment(args.anthology_id, args.path, args.attach_type, overwrite=False)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "anthology_id", help="The Anthology paper ID to revise (e.g., P18-1001)"
    )
    parser.add_argument(
        "path", type=str, help="Path to the revised attachment ID (can be URL)"
    )
    parser.add_argument(
        "attach_type", type=str, default='Supplementary', help="attachment type"
    )

    parser.add_argument(
        "--attachment-root",
        default=os.path.join(os.environ["HOME"], "anthology-files/attachments"),
        help="Anthology web directory root.",
    )
    args = parser.parse_args()

    main(args)


# bin/add_award.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Used to add an award to a paper in the Anthology.

Usage:

  add_award.py paper_id "Award title"

If you have them all listed in a file, "awards.txt", like this:

    2021.naacl-main.119 Best Long Paper
    2021.naacl-main.31 Outstanding Long Paper
    2021.naacl-main.185 Outstanding Long Paper
    2021.naacl-main.410 Best Short Paper
    2021.naacl-main.208 Outstanding Short Paper
    2021.naacl-main.51 Best Thematic Paper
    2021.naacl-industry.20 Best Industry Paper

You can do

    cat awards.txt | while read line; do
      id=$(echo $line | cut -d" " -f1)
      text=$(echo $line | cut -d" " -f2-)
      ./bin/add_award.py $id "$text"
    done

The commit the changes and push.
"""

import argparse
import filetype
import os
import shutil
import ssl
import sys
import tempfile

from anthology.utils import (
    deconstruct_anthology_id,
    make_simple_element,
    indent,
    compute_hash,
    infer_url,
    is_newstyle_id,
)
from anthology.data import PDF_LOCATION_TEMPLATE

import lxml.etree as ET


def main(args):
    print(f"Adding {args.award} to {args.anthology_id}...")

    collection_id, volume_id, paper_id = deconstruct_anthology_id(args.anthology_id)

    # Update XML
    xml_file = os.path.join(
        os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
    )
    tree = ET.parse(xml_file)
    paper = tree.getroot().find(f"./volume[@id='{volume_id}']/paper[@id='{paper_id}']")
    if paper is None:
        print(f"Error: Can't find paper {args.anthology_id}, quitting")

    existing_award = paper.find("./award")
    if existing_award is not None and award.text.lower() == args.award:
        print(
            f"Error: Award {args.award} already exists for {args.anthology_id}, quitting"
        )

    make_simple_element("award", args.award, parent=paper)
    indent(tree.getroot())

    tree.write(xml_file, encoding="UTF-8", xml_declaration=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "anthology_id", help="The Anthology paper ID to revise (e.g., P18-1001)"
    )
    parser.add_argument("award", help="Brief description of the changes.")
    args = parser.parse_args()

    main(args)


# bin/add_bulk_attachments.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Used to add attachments to the Anthology.
Usage:

    add_attachment_bulk.py CSV_FILE

Where CSV_FILE is output from the Microsoft form (https://forms.office.com/Pages/ResponsePage.aspx?id=DQSIkWdsW0yxEjajBLZtrQAAAAAAAAAAAAMAABqTSThUN0I2VEdZMTk4Sks3S042MVkxUEZQUVdOUS4u) we use to collect attachments, and has the following headers:

    ID,Start time,Completion time,Email,Name,Anthology ID,URL where we can download the attachment,Attachment type,"For corrections or errata, please explain in detailed prose what has changed.",Your name,Your email address,I agree to the Anthology's CC-BY-4.0 distribution license.

Downloads the files, edits the XML, and dumps a log to
add_attachments.log, along with emails to be sent to those whose
imports failed.
"""

import argparse
import csv
import filetype
import os
import shutil
import ssl
import sys
import tempfile

from anthology.utils import (
    build_anthology_id,
    deconstruct_anthology_id,
    indent,
    compute_hash,
)

import lxml.etree as ET
import urllib.request

ALLOWED_TYPES = ["pdf", "pptx", "zip"]
ATTACHMENT_TYPES = "Poster Presentation Note Software Supplementary Dataset".split()


def add_attachment(anthology_id, path, attach_type, overwrite=False):
    """
    Adds a single attachment to the Anthology data files.

    Arguments:
    - The ACL ID of the paper (e.g., P17-1012)
    - The path to the attachment (can be a URL)
    - The attachment type (poster, presentation, note, software)
    - Whether to overwrite the downloaded file.
    """

    collection_id, volume_id, paper_id = deconstruct_anthology_id(anthology_id)

    if path.startswith("http"):
        _, input_file_path = tempfile.mkstemp()
        try:
            print(
                f"-> Downloading file from {path} to {input_file_path}", file=sys.stderr
            )
            request = urllib.request.Request(path, headers={'User-Agent': 'Mozilla/5.0'})
            with urllib.request.urlopen(request) as url, open(
                input_file_path, mode="wb"
            ) as input_file_fh:
                input_file_fh.write(url.read())
        except ssl.SSLError:
            raise Exception(f"Could not download {path}")
        except Exception as e:
            raise e
    else:
        input_file_path = path

    file_extension = path.replace("?dl=1", "").split(".")[-1]
    # Many links from file sharing services are not informative and don't have
    # extensions, so we could try to guess.
    if file_extension not in ALLOWED_TYPES:
        detected = filetype.guess(input_file_path)
        if detected is not None:
            file_extension = detected.mime.split("/")[-1]
            if file_extension not in ALLOWED_TYPES:
                print(
                    f"Could not determine file extension for {anthology_id} at {path}",
                    file=sys.stderr,
                )

    with open(input_file_path, "rb") as f:
        checksum = compute_hash(f.read())

    # Update XML
    xml_file = os.path.join(
        os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
    )
    tree = ET.parse(xml_file)

    attachment_file_name = f"{anthology_id}.{attach_type}.{file_extension}"

    paper = tree.getroot().find(f"./volume[@id='{volume_id}']/paper[@id='{paper_id}']")
    if paper is not None:
        # Check if attachment already exists
        for attachment in paper.findall("attachment"):
            if attachment.text == attachment_file_name:
                print(
                    f"-> attachment {attachment_file_name} already exists in the XML",
                    file=sys.stderr,
                )
                break
        else:
            attachment = ET.Element("attachment")
            attachment.attrib["type"] = attach_type.lower()
            attachment.attrib["hash"] = checksum
            attachment.text = attachment_file_name

            paper.append(attachment)
            indent(tree.getroot())
            tree.write(xml_file, encoding="UTF-8", xml_declaration=True)
            print(
                f"-> added attachment {attachment_file_name} to the XML", file=sys.stderr
            )

    else:
        print(f"Paper {anthology_id} not found in the Anthology", file=sys.stderr)

    # Make sure directory exists
    output_dir = os.path.join(args.attachment_root, collection_id[0], collection_id)
    if not os.path.exists(output_dir):
        #        print(f"-> Creating directory {output_dir}", file=sys.stderr)
        os.makedirs(output_dir)

    # Copy file
    dest_path = os.path.join(output_dir, attachment_file_name)
    if os.path.exists(dest_path) and not overwrite:
        print(
            f"-> target file {dest_path} already in place, refusing to overwrite",
            file=sys.stderr,
        )
        return None

    shutil.copy(input_file_path, dest_path)
    os.chmod(dest_path, 0o644)
    print(f"-> copied {input_file_path} to {dest_path} and fixed perms", file=sys.stderr)

    # Clean up
    if path.startswith("http"):
        os.remove(input_file_path)

    return dest_path


def main(args):
    attachments = {}
    with open(args.csv_file) as csv_file:
        for row in csv.DictReader(csv_file):
            # ID,Start time,Completion time,Email,Name,Anthology ID,URL where we can download the attachment,Attachment type,"For corrections or errata, please explain in detailed prose what has changed.",Your name,Your email address,I agree to the Anthology's CC-BY-4.0 distribution license
            anthology_id = row["Anthology ID"].strip()
            download_path = row["URL"]
            attachment_type = row["Attachment type"]
            submitter_name = row["Your name"]
            submitter_email = row["Your email address"]
            submitted = row["Completion time"]

            if attachment_type not in ATTACHMENT_TYPES:
                print(
                    f"{anthology_id}: Skipping unknown type {attachment_type}: {download_path}",
                    file=sys.stderr,
                )
                continue

            if anthology_id in attachments:
                print(
                    f"{anthology_id}: Received multiple entries, only processing the last one ({attachment_type}): {download_path}",
                    file=sys.stderr,
                )

            attachments[anthology_id] = (
                download_path,
                attachment_type,
                submitter_name,
                submitter_email,
                submitted,
            )

    succeeded = 0
    failed = 0
    with open(args.logfile, "a") as log:
        for anthology_id, (path, attach_type, name, email, date) in attachments.items():
            try:
                print(f"Processing attachment for {anthology_id}", file=sys.stderr)
                success = add_attachment(
                    anthology_id, path, attach_type, overwrite=args.overwrite
                )
                if success:
                    succeeded += 1
                    print(f"{anthology_id}: SUCCESS.", file=log)
                else:
                    print(f"{anthology_id}: ALREADY DONE (use -o to redo).", file=log)
            except Exception as reason:
                failed += 1
                print(f"{anthology_id}: FAILURE", file=log)
                with open(f"{args.logfile}.{anthology_id}.txt", "w") as email_log:
                    print(
                        f"{email}\n"
                        f"ACL Anthology: failed to add attachment for {anthology_id}\n"
                        f"Dear {name},\n"
                        f"\n"
                        f"On {date} you submitted the following attachment to the ACL Anthology\n"
                        f"\n"
                        f"  paper ID: {anthology_id}\n"
                        f"      link: {path}\n"
                        f"\n"
                        f"Adding this attachment failed. The reason reported was:\n"
                        f"\n"
                        f"  {reason}\n"
                        f"\n"
                        f"To resubmit, follow the instructions here:\n"
                        f"\n"
                        f"  https://aclanthology.org/info/corrections/\n",
                        f"\n"
                        f"There is no need to respond to this email.\n"
                        f"\n"
                        f"Sincerely,\n"
                        f"Matt Post\n"
                        f"Anthology Director\n",
                        file=email_log,
                    )

    print(
        f"Processed {len(attachments)} attachments ({succeeded} succeeded, {failed} failed)."
    )
    print(f"Wrote logfile to {args.logfile}.")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("csv_file", help="The CSV file from the Microsoft form")
    parser.add_argument(
        "--overwrite", "-o", action="store_true", help="Overwrite attachments"
    )
    parser.add_argument(
        "--logfile", "-l", default="add_attachments.log", help="Logfile to write to"
    )
    parser.add_argument(
        "--attachment-root",
        "-d",
        default=os.path.join(os.environ["HOME"], "anthology-files/attachments"),
        help="Anthology web directory root.",
    )
    args = parser.parse_args()

    main(args)


# bin/add_dois.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Min-Yen Kan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Used to add ingested DOIs into the Anthology XML.
Does not actually assign DOIs (separate script to manufacture XML to submit to Crossref), but
simply adds to the XML, after checking that the DOI URL exists and resolves.

Usage:

    add_dois.py [list of volume IDs]

e.g.,

    python3 add_dois.py P19-1 P19-2 P19-3 P19-4 W19-32

Modifies the XML.  Warns if DOIs already present.  Use -f to force.

Limitations:
- Doesn't current add the entire proceedings volume itself.
"""

import sys
import os
import tempfile
import anthology.data as data

from anthology.utils import (
    build_anthology_id,
    deconstruct_anthology_id,
    stringify_children,
    test_url_code,
    indent,
    make_simple_element,
)
from anthology.formatter import MarkupFormatter
from itertools import chain
from time import sleep

import lxml.etree as ET


def add_doi(xml_node, collection_id, volume_id, force=False):
    if "id" in xml_node.attrib:
        # normal paper
        paper_id = int(xml_node.attrib["id"])
    else:
        # frontmatter
        paper_id = 0

    anth_id = build_anthology_id(collection_id, volume_id, paper_id)
    new_doi_text = f"{data.DOI_PREFIX}{anth_id}"

    doi = xml_node.find("doi")
    if doi is not None:
        print(
            f"-> [{anth_id}] Cowardly refusing to overwrite existing DOI {doi.text} (use --force)",
            file=sys.stderr,
        )
        return False

    doi_url = f"{data.DOI_URL_PREFIX}{data.DOI_PREFIX}{anth_id}"
    for tries in [1, 2, 3]:  # lots of random failures
        result = test_url_code(doi_url)
        if result.status_code == 200:
            doi = make_simple_element("doi", text=new_doi_text)
            print(f"-> Adding DOI {new_doi_text}", file=sys.stderr)
            xml_node.append(doi)
            return True
        elif result.status_code == 429:  # too many requests
            pause_for = int(result.headers["Retry-After"])
            print(f"--> Got 429, pausing for {pause_for} seconds", file=sys.stderr)
            sleep(pause_for + 1)
        elif result.status_code == 404:  # not found
            print(f"--> Got 404", file=sys.stderr)
            break
        else:
            print(f"--> Other problem: {result}", file=sys.stderr)

    print(f"-> Couldn't add DOI for {doi_url}", file=sys.stderr)
    return False


def process_volume(anthology_volume):

    collection_id, volume_id, _ = deconstruct_anthology_id(anthology_volume)

    print(f"Attempting to add DOIs for {anthology_volume}", file=sys.stderr)

    # Update XML
    xml_file = os.path.join(
        os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
    )
    tree = ET.parse(xml_file)

    formatter = MarkupFormatter()

    num_added = 0

    volume = tree.getroot().find(f"./volume[@id='{volume_id}']")
    if volume is not None:
        volume_booktitle = volume.find(f"./meta/booktitle")
        volume_title = formatter.as_text(volume_booktitle)
        print(f'-> Found volume "{volume_title}"', file=sys.stderr)

        # Iterate through all papers
        for paper in chain(volume.find("frontmatter"), volume.findall("paper")):
            added = add_doi(paper, collection_id, volume_id, force=args.force)
            if added:
                num_added += 1
                sleep(1)

        indent(tree.getroot())

        tree.write(xml_file, encoding="UTF-8", xml_declaration=True)
        print(
            f"-> added {num_added} DOIs to to the XML for collection {collection_id}",
            file=sys.stderr,
        )

    else:
        print(f"-> FATAL: volume {volume} not found in the Anthology", file=sys.stderr)
        sys.exit(1)


def main(args):
    for volume in args.anthology_volumes:
        process_volume(volume)


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "anthology_volumes",
        nargs="+",
        help="One or more Anthology volumes (e.g., P17-1, W18-02)",
    )
    parser.add_argument(
        "--prefix",
        "-p",
        default=data.DOI_PREFIX,
        help="The DOI prefix to use (default: " + data.DOI_PREFIX + ")",
    )
    parser.add_argument(
        "--force",
        "-f",
        help="Force overwrite of existing DOI information",
        action="store_true",
    )
    args = parser.parse_args()

    main(args)


# bin/add_isbns.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Usage:
  add_isbns.py ISBN-file

  Adds <language>eng</language> to all files in a specified XML file.
  Created for LREC 2020; kept for later adaptation.
"""

import argparse
import os
import sys

from anthology.utils import (
    make_simple_element,
    indent,
)

import lxml.etree as ET


def main(args):

    for line in args.isbn_file:
        venue, isbn = line.rstrip().split()

        xml_file = os.path.join(
            os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{venue}.xml"
        )
        if not os.path.exists(xml_file):
            print(f"Can't find {xml_file}")
            continue
        tree = ET.parse(xml_file)
        meta = tree.getroot().find(f".//volume[@id='1']/meta")
        if meta is not None and meta.find("./isbn") is None:
            print(f"Adding {isbn} to {venue} meta block")
            make_simple_element("isbn", isbn, parent=meta)
        elif volume.find("./isbn") is not None:
            print(f"{venue} already done")

        indent(tree.getroot())
        tree.write(xml_file, encoding="UTF-8", xml_declaration=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("isbn_file", type=argparse.FileType("r"))
    args = parser.parse_args()

    main(args)


# bin/add_language.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Usage:
  add_language.py XML_file

  Adds <language>eng</language> to all files in a specified XML file.
  Created for LREC 2020; kept for later adaptation.
"""

import argparse

from anthology.utils import (
    make_simple_element,
    indent,
)

import lxml.etree as ET


def main(args):

    for xml_file in args.xml_files:
        tree = ET.parse(xml_file)
        for paper in tree.getroot().findall(f".//paper"):
            make_simple_element("language", "eng", parent=paper)

        indent(tree.getroot())
        tree.write(xml_file, encoding="UTF-8", xml_declaration=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("xml_files", nargs="+")
    args = parser.parse_args()

    main(args)


# bin/add_missing_attachments.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
One-time script used to add missing attachments to the XML.

Collect files:

    find attachments -type f > missing.txt

Process:

    cat missing.txt | python3 add_missing_attachments.py
"""

import argparse
import os
import shutil
import ssl
import sys
import tempfile

from anthology.utils import build_anthology_id, deconstruct_anthology_id, indent

import lxml.etree as ET
import urllib.request

ALLOWED_TYPES = ["pdf", "pptx", "zip"]
ATTACHMENT_TYPES = "Poster Presentation Note Software Supplementary".split()


def main(args):

    for lineno, line in enumerate(sys.stdin, 1):
        # attachments/D/D15/D15-1272.Attachment.pdf
        tokens = line.rstrip().split("/")
        attachment_file_name = tokens[-1]
        try:
            anth_id, kind, *rest = attachment_file_name.split(".")
        except:
            print(f"Couldn't parse file {attachment_file_name} into 3 pieces")
            continue

        try:
            collection_id, volume_id, paper_id = deconstruct_anthology_id(anth_id)
        except:
            print(f"[{lineno}] BAD LINE {line.rstrip()}")

        # Update XML
        xml_file = os.path.join(
            os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
        )
        tree = ET.parse(xml_file)

        if int(paper_id) == 0:
            paper = tree.getroot().find(f"./volume[@id='{volume_id}']/frontmatter")
        else:
            paper = tree.getroot().find(
                f"./volume[@id='{volume_id}']/paper[@id='{paper_id}']"
            )
        if paper is not None:
            # Check if attachment already exists
            for attachment in paper.findall("attachment"):
                if attachment.text == attachment_file_name:
                    #                    print(f'-> attachment {attachment_file_name} already exists in the XML', file=sys.stderr)
                    break
            else:
                attachment = ET.Element("attachment")
                attachment.attrib["type"] = kind.lower()
                attachment.text = attachment_file_name

                paper.append(attachment)
                indent(tree.getroot())
                tree.write(xml_file, encoding="UTF-8", xml_declaration=True)
                print(
                    f"-> [{lineno}] added attachment {attachment_file_name} to the XML",
                    file=sys.stderr,
                )

        else:
            print(
                f"-> FATAL: [{lineno}] paper ({anth_id}) not found in the Anthology",
                file=sys.stderr,
            )
            sys.exit(1)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    args = parser.parse_args()

    main(args)


# bin/add_revision.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Used to add revisions to the Anthology.
Assumes all files have a base format like ANTHOLOGY_ROOT/P/P18/P18-1234.pdf format.
The revision process is as follows.

- The original paper is named as above.
- When a first revision is created, the original paper is archived to PYY-XXXXv1.pdf.
- The new revision is copied to PYY-XXXXvN, where N is the next revision ID (usually 2).
  The new revision is also copied to PYY-XXXX.pdf.
  This causes it to be returned by the anthology when the base paper format is queried.

Usage:

  add_revision.py [-e] paper_id URL_OR_PATH.pdf "Short explanation".

`-e` denotes erratum instead of revision.
By default, a dry run happens.
When you are ready, add `--do`.
"""

import argparse
import filetype
import os
import shutil
import ssl
import sys
import tempfile

from anthology.utils import (
    deconstruct_anthology_id,
    make_simple_element,
    indent,
    compute_hash_from_file,
    infer_url,
    is_newstyle_id,
    retrieve_url,
    get_pdf_dir,
    get_xml_file,
)
from anthology.data import (
    PDF_LOCATION_TEMPLATE,
    ANTHOLOGY_FILE_DIR,
)

import lxml.etree as ET
import urllib.request

from datetime import datetime


def validate_file_type(path):
    """Ensure downloaded file mime type matches its extension (e.g., PDF)"""
    detected = filetype.guess(path)
    if detected is None or not detected.mime.endswith(detected.extension):
        mime_type = 'UNKNOWN' if detected is None else detected.mime
        print(
            f"FATAL: file {path} has MIME type {mime_type}",
            file=sys.stderr,
        )
        sys.exit(1)


def add_revision(
    anth_id, pdf_path, explanation, change_type="revision", dry_run=True, date=None
):
    """
    Takes an Anthology ID. It then adds a revision to the Anthology XML,
    updating and writing the XML file, and copies the PDFs into place.
    For PDFs, the revised PDF is saved to {anth_id}.pdf and {anth_id}v{version}.pdf.
    For the first revision, we first copy {anth_id}.pdf to {anth_id}v1.pdf.
    """
    if date is None:
        now = datetime.now()
        date = f"{now.year}-{now.month:02d}-{now.day:02d}"

    def maybe_copy(file_from, file_to):
        if not dry_run:
            print("-> Copying from {} -> {}".format(file_from, file_to), file=sys.stderr)
            shutil.copy(file_from, file_to)
            os.chmod(file_to, 0o644)
        else:
            print(
                "-> DRY RUN: Copying from {} -> {}".format(file_from, file_to),
                file=sys.stderr,
            )

    # The new version
    revno = None

    change_letter = "e" if change_type == "erratum" else "v"

    checksum = compute_hash_from_file(pdf_path)

    # Files for old-style IDs are stored under anthology-files/pdf/P/P19/*
    # Files for new-style IDs are stored under anthology-files/pdf/2020.acl/*
    output_dir = get_pdf_dir(anth_id)

    # Make sure directory exists
    if not os.path.exists(output_dir):
        print(f"-> Creating directory {output_dir}", file=sys.stderr)
        os.makedirs(output_dir)

    canonical_path = os.path.join(output_dir, f"{anth_id}.pdf")

    # Update XML
    xml_file = get_xml_file(anth_id)
    collection_id, volume_id, paper_id = deconstruct_anthology_id(anth_id)
    tree = ET.parse(xml_file)
    if paper_id == "0":
        paper = tree.getroot().find(f"./volume[@id='{volume_id}']/frontmatter")
    else:
        paper = tree.getroot().find(
            f"./volume[@id='{volume_id}']/paper[@id='{paper_id}']"
        )
    if paper is not None:
        revisions = paper.findall(change_type)
        revno = 1 if change_type == "erratum" else 2
        for revision in revisions:
            revno = int(revision.attrib["id"]) + 1

        if not dry_run:
            # Update the URL hash on the <url> tag
            url = paper.find("./url")
            if url is not None:
                url.attrib["hash"] = checksum

            if change_type == "revision" and revno == 2:
                if paper.find("./url") is not None:
                    current_version_url = infer_url(paper.find("./url").text) + ".pdf"

                # Download original file
                # There are no versioned files the first time around, so create the first one
                # (essentially backing up the original version)
                revised_file_v1_path = os.path.join(
                    output_dir, f"{anth_id}{change_letter}1.pdf"
                )

                retrieve_url(current_version_url, revised_file_v1_path)
                validate_file_type(revised_file_v1_path)

                old_checksum = compute_hash_from_file(revised_file_v1_path)

                # First revision requires making the original version explicit
                revision = make_simple_element(
                    change_type,
                    None,
                    attrib={
                        "id": "1",
                        "href": f"{anth_id}{change_letter}1",
                        "hash": old_checksum,
                    },
                    parent=paper,
                )

            revision = make_simple_element(
                change_type,
                explanation,
                attrib={
                    "id": str(revno),
                    "href": f"{anth_id}{change_letter}{revno}",
                    "hash": checksum,
                    "date": date,
                },
                parent=paper,
            )
            indent(tree.getroot())

            tree.write(xml_file, encoding="UTF-8", xml_declaration=True)
            print(
                f'-> Added {change_type} node "{revision.text}" to XML', file=sys.stderr
            )

    else:
        print(
            f"-> FATAL: paper ID {anth_id} not found in the Anthology",
            file=sys.stderr,
        )
        sys.exit(1)

    revised_file_versioned_path = os.path.join(
        output_dir, f"{anth_id}{change_letter}{revno}.pdf"
    )

    # Copy the file to the versioned path
    maybe_copy(pdf_path, revised_file_versioned_path)

    # Copy it over the canonical path
    if change_type == "revision":
        maybe_copy(pdf_path, canonical_path)


def main(args):
    change_type = "erratum" if args.erratum else "revision"

    print(f"Processing {change_type} to {args.anthology_id}...")

    # TODO: make sure path exists, or download URL to temp file
    if args.path.startswith("http"):
        _, input_file_path = tempfile.mkstemp()
        retrieve_url(args.path, input_file_path)
    else:
        input_file_path = args.path

    validate_file_type(input_file_path)

    add_revision(
        args.anthology_id,
        input_file_path,
        args.explanation,
        change_type=change_type,
        dry_run=args.dry_run,
    )

    if args.path.startswith("http"):
        os.remove(input_file_path)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "anthology_id", help="The Anthology paper ID to revise (e.g., P18-1001)"
    )
    parser.add_argument(
        "path", type=str, help="Path to the revised paper ID (can be URL)"
    )
    parser.add_argument("explanation", help="Brief description of the changes.")
    parser.add_argument(
        "--erratum",
        "-e",
        action="store_true",
        help="This is an erratum instead of a revision.",
    )
    now = datetime.now()
    today = f"{now.year}-{now.month:02d}-{now.day:02d}"
    parser.add_argument(
        "--date",
        "-d",
        type=str,
        default=today,
        help="The date of the revision (ISO 8601 format)",
    )
    parser.add_argument(
        "--dry-run", "-n", action="store_true", default=False, help="Just a dry run."
    )
    args = parser.parse_args()

    main(args)


# bin/add_venue.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Adds a venue to the data/yaml/venues.yaml file.
Usage:

    add_venue.py acronym "name" [--url URL] [--acl]
"""

import argparse
import os
import sys

from slugify import slugify

from anthology.venues import VenueIndex


def main(args):
    datadir = os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
    venues = VenueIndex(srcdir=datadir)

    print(f"Adding '{args.acronym}' ({args.name})")
    venues.add_venue(args.acronym, args.name, is_acl=args.acl, url=args.url)

    venues.dump(datadir)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("acronym", help="Venue acronym (e.g., BlackboxNLP)")
    parser.add_argument(
        "name",
        help="Venue name (e.g., Workshop on analyzing and interpreting neural networks for NLP)",
    )
    parser.add_argument("--acl", action="store_true", help="Venue is an ACL venue")
    parser.add_argument("--url", help="Venue URL")
    args = parser.parse_args()

    main(args)


# bin/add_video_tags.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Namratha Urs <namrathaurs@my.unt.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This script is used to add video tags to the Anthology towards ingestion of videos.

Usage:
    add_video_tags.py TSV_files

        where TSV_files are the tab-separated values (TSV) files containing the tuples (anthology_id, presentation_id)

    Consolidates all the TSV files passed to the script, edits the XML by adding a properly-indented video tag to the
    end of the <paper> element and rewrites the XML.
"""

import pandas as pd
import os
import lxml.etree as et
import argparse

from anthology.utils import deconstruct_anthology_id, make_simple_element, indent

root = os.path.dirname(os.path.abspath(__file__))
data_dir = os.path.join(root, "../data/xml")


def combine_tsv(files):
    combined_df = pd.concat(
        [pd.read_csv(fname, keep_default_na=False, sep="\t") for fname in files]
    )
    combined_df = combined_df[
        (combined_df.anthology_id != "") & (combined_df.anthology_id != "nan")
    ]
    return combined_df


def split_anth_id(anth_id):
    coll_id, _, _ = deconstruct_anthology_id(anth_id)
    return coll_id


def add_video_tag(anth_paper, xml_parse):
    coll_id, vol_id, paper_id = deconstruct_anthology_id(anth_paper.anthology_id)
    paper = xml_parse.find(f'./volume[@id="{vol_id}"]/paper[@id="{paper_id}"]')

    if anth_paper.presentation_id.startswith("http"):
        video_url = anth_paper.presentation_id
    else:
        video_url = "https://slideslive.com/{}".format(anth_paper.presentation_id)

    make_simple_element("video", attrib={"href": video_url}, parent=paper)


def main(args):
    combo_df = combine_tsv(args['tsv_files'])
    combo_df_uniques = combo_df['anthology_id'].apply(split_anth_id).unique()

    for xml in os.listdir(data_dir):
        fname, ext = os.path.splitext(xml)
        if fname in combo_df_uniques.tolist() or fname == "2020.acl":
            tree = et.parse(os.path.join(data_dir, xml))

            df_subset = combo_df[combo_df['anthology_id'].str.startswith(fname)]
            df_subset.apply(add_video_tag, axis=1, xml_parse=tree)

            with open(os.path.join(data_dir, fname + ".xml"), 'wb') as f:
                indent(tree.getroot())
                tree.write(f, encoding="UTF-8", xml_declaration=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Adds video tags to the anthology XML.')
    parser.add_argument(
        'tsv_files',
        nargs='+',
        help='Two-column TSV containing (anthology_id, presentation_id)',
    )

    cl_args = parser.parse_args()
    if cl_args == 0:
        parser.print_help()
    else:
        main(
            vars(cl_args)
        )  # vars converts the argparse's Namespace object to a dictionary


# bin/author_case.py

#!/usr/bin/env python3

"""author_case.py

Try to correct author names that are written in all uppercase or all lowercase.

usage: author_case.py

Writes to stdout a list of changes that can be read by
change_authors.py. The user can edit this list before giving it to
change_authors.py to actually apply the changes.
"""

import sys
import os.path
import anthology
import logging
import re

logging.basicConfig(level=logging.INFO)

# Two-letter words that should be changed from upper->titlecase
two_letter = [
    # Mandarin Pinyin
    # https://gist.github.com/stevejackson/1429696
    'ba pa ma fa da ta na la ga ka ha za ca sa',
    'za ca sa',
    'ai',
    'an',
    'ao',
    'me de te ne le ge ke he re ce se',
    'ei',
    'er',
    'yi bi pi mi di ti ni li ji qi xi ri zi ci si',
    'ya ye yo yu',
    'bo po mo fo lo',
    'ou',
    'wu bu pu mu fu du tu nu lu gu ku hu ru zu cu su',
    'wa wo',
    'nv lv ju qu',
    # Cantonese
    'ng',
]
two_letter = set().union(*[s.split() for s in two_letter])

# Words that should not be changed from upper->titlecase
upper = set('III PVS GSK'.split())


def normalize(s):
    if s == s.lower():
        s = s.title()
    else:
        words = re.split(r'([ .-])', s)
        for i in range(0, len(words), 2):
            w = words[i]
            if w == w.upper() and (
                len(w) >= 3 and w not in upper or len(w) == 2 and w.lower() in two_letter
            ):
                w = w.title()
            words[i] = w
        s = ''.join(words)
    return s


if __name__ == "__main__":
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    datadir = os.path.join(scriptdir, '..', 'data')
    anth = anthology.Anthology(importdir=datadir)
    for paperid, paper in anth.papers.items():
        for role in ['author', 'editor']:
            if role in paper.attrib:
                for name, personid in paper.attrib[role]:
                    # Many mononyms are corporate authors containing
                    # acronyms; better not to touch these
                    if name.first is None or name.first == '':
                        continue
                    first_norm = normalize(name.first)
                    last_norm = normalize(name.last)
                    if name.first != first_norm or name.last != last_norm:
                        print(
                            '{}\t{}\t{} || {}\t{} || {}'.format(
                                paperid,
                                role,
                                name.first,
                                name.last,
                                first_norm,
                                last_norm,
                            )
                        )


# bin/author_names_to_variants.py

#!/usr/bin/env python3
'''
Usage example: python bin/author_names_to_variants.py data/xml/2021.ccl.xml data/xml/2021.ccl.xml

Input: xml file generated by ingest.py
Output: updated xml file including name variant
'''
import lxml.etree as etree
import sys
import unicodedata

tree = etree.parse(sys.argv[1])
root = tree.getroot()
if not root.tail:
    root.tail = "\n"


def guess_script(s):
    n = len(s)
    n_latin = len([c for c in s if 'a' <= c.lower() <= 'z'])
    n_han = len([c for c in s if '\u3400' <= c <= '\u9ffc'])
    if n_latin / n > 0.5:
        return 'latn'
    if n_han / n > 0.5:
        return 'hani'


def process(names):
    scripts = [
        guess_script(''.join(part.text for part in name if part.tag in ['first', 'last']))
        for name in names
    ]
    variants = None
    if len(scripts) % 2 == 0:
        n = len(scripts) // 2
        if all(script == 'latn' for script in scripts[:n]) and all(
            script == 'hani' for script in scripts[n:]
        ):
            primaries = names[:n]
            variants = names[n:]
        elif all(script == 'hani' for script in scripts[:n]) and all(
            script == 'latn' for script in scripts[n:]
        ):
            primaries = names[n:]
            variants = names[:n]

    if variants is not None:
        for i in reversed(range(n)):
            name = variants[i]
            names.remove(name)
            name.tag = 'variant'
            name.set('script', scripts[n + i])
            primaries[i].append(name)
    else:
        print(
            f'skipping:',
            ' '.join(etree.tostring(name, encoding='unicode') for name in names),
        )


for paper in root.findall(".//paper"):
    process(list(paper.findall("author")))
for meta in root.findall(".//meta"):
    process(list(meta.findall("editor")))

tree.write(sys.argv[2], encoding='UTF-8', xml_declaration=True)


# bin/auto_authors.py

"""
Try to automatically restore accents in author names by downloading and scraping the PDFs.
Reads and writes Anthology XML files.

To do:
- Update for new hierarchical XML structure
- Instead of making changes, output a list of changes that can be read by change_authors.py.
"""

import tika.parser
import requests
import sys
import lxml.etree as etree
import re
import unicodedata
import os.path, glob
import anthology
import copy
import collections
import time
from normalize_anth import clean_unicode, curly_quotes
import yaml, yamlfix


def guess_url(paper):
    volume = paper.getparent()

    if (
        paper.attrib["id"].endswith("000")
        or volume.attrib["id"].startswith("W")
        and paper.attrib["id"].endswith("00")
    ):
        return None

    if args.pdfdir:
        path = os.path.join(
            args.pdfdir,
            volume.attrib["id"][0],
            volume.attrib["id"],
            "{}-{}.pdf".format(volume.attrib["id"], paper.attrib["id"]),
        )
        print(path)
        if os.path.exists(path):
            return "file:" + path

    url = paper.find("url") is not None and paper.find("url").text
    if not url:
        url = paper.find("href") and paper.find("href").text
    if not url:
        url = paper.attrib.get("href", None)
    if not url:
        url = "http://www.aclanthology.org/{}-{}".format(
            volume.attrib["id"], paper.attrib["id"]
        )

    return url


def slugify(s):
    # Bug: Foobar and Foo Bar have different slugs.
    # We could remove spaces,
    # but there are many PDFs that have extra spaces, and we don't want those.

    # Split on camelCase
    s = re.sub(r"([a-z])([A-Z])", r"\1 \2", s)

    # Split on hyphens
    s = s.replace("-", " ")

    # Remove accents and symbols
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if c.isalpha() or c.isspace())

    s = s.lower()
    s = "-".join(s.split())

    return s


def is_namechar(c):
    # hyphen can occur in names, but not at beginning or end,
    # so we don't include it her
    return (
        c in ".“”‘’«»„"
        or unicodedata.category(c)[0] not in "CNPSZ"
        and not (0x370 <= ord(c) < 0x400)  # Greek letters used as symbols
    )


email_re = r"(\{.*\}|\S+)\@\S+\.\S+"

delay = 0.0


def get_url(url, retries=10):
    global delay
    if url.startswith("http:") or url.startswith("https:"):
        if retries == 0:
            logger.error("max retries exceeded; skipping")
            return {}
        logger.info("getting {}".format(url))
        try:
            time.sleep(delay)
            r = requests.get(url, timeout=10)
            if not r:
                logger.error("could not download PDF")
                return None
            content = r.content
        except KeyboardInterrupt:
            raise
        except requests.exceptions.Timeout:
            delay += 1.0
            logger.warning("connection timed out; increasing delay to {} s".format(delay))
            return get_url(url, retries - 1)
        except Exception as e:
            logger.error(str(e))
            return get_url(url, retries - 1)
    elif url.startswith("file:"):
        file = url[5:]
        logger.info("reading file {}".format(file))
        content = open(file, "rb").read()
    else:
        assert False

    return content


def scrape_authors(content, retries=10):
    try:
        raw = tika.parser.from_buffer(content)
    except KeyboardInterrupt:
        raise
    except Exception as e:
        logger.error(str(e))
        return scrape_authors(content, retries - 1)

    text = raw.get("content", None)
    if text is None:
        return {}
    index = collections.defaultdict(list)
    li = 0
    for line in text.splitlines():
        # some papers have spaces between every letter
        nospace = "".join(line.split())
        if nospace == "Abstract":
            break
        if nospace == "":
            continue

        li += 1
        if li > 50:
            break

        line = clean_unicode(line)
        line = curly_quotes(line)
        line = unicodedata.normalize(
            "NFKC", line
        )  # more aggressive normalization than clean_unicode; for example, in an author name, 𝐦 (mathematical bold small m) is presumably a PDF bug
        logger.info("line:  " + line)

        # delete email addresses, so that they are not mistaken for names
        for m in re.finditer(email_re, line):
            logger.info("deleting email address {}".format(m.group(0)))
        line = re.sub(email_re, "", line)

        line = line.replace(",", ", ")

        words = []
        slugs = []
        for word in line.split():
            word = word.strip()

            # strip leading and trailing symbols
            while len(word) > 0 and not is_namechar(word[0]):
                word = word[1:]
            while len(word) > 0 and not is_namechar(word[-1]):
                word = word[:-1]

            if word == "":
                continue

            words.append(word)
            slugs.append(slugify(word))

        # Index up to 4-grams
        for g in range(1, 5):
            for i in range(len(words) - g + 1):
                index["-".join(slugs[i : i + g])].append(" ".join(words[i : i + g]))

    return index


def change(index, name):
    if name == "":
        return name
    slug = slugify(name)
    if slug in index:
        if len(set(index[slug])) > 1:
            logger.warning("name collision: {}".format(", ".join(index[slug])))
        newname = index[slug][0]
        if not name.isupper() and newname.isupper():
            # this could backfire for initials, like Abc -> ABC
            logger.info("not uppercasing: {} -> {}".format(name, newname))
            return name
        if not name.islower() and newname.islower():
            # an all-lowercase name is likely to be an email address
            logger.info("not lowercasing: {} -> {}".format(name, newname))
            return name
        if newname != name:
            logger.info("changing: {} -> {}".format(name, newname))
        return newname
    else:
        logger.warning("name {} in XML but not in PDF".format(name))
        return name


def merge_people(variants, can1, can2):
    if can1 == can2:
        return
    # This is really inefficient, but it doesn't actually happen that much
    i1 = i2 = None
    for i, d in enumerate(variants):
        can = anthology.people.PersonName.from_dict(d["canonical"])
        if can == can1:
            i1 = i
        if can == can2:
            i2 = i
    if i1 is not None and i2 is not None:
        logger.error(
            "Please manually merge '{}' and '{}' in name_variants.yaml".format(can1, can2)
        )
        return
    elif i1 is not None:
        i = i1
        new = can2
    elif i2 is not None:
        i = i2
        new = can1
    else:
        # choose can2 to be canonical, since it is the new and hopefully more correct name.
        variants.append({"canonical": {"first": can2.first, "last": can2.last}})
        i = len(variants) - 1
        new = can1
    for v in variants[i].get("variants", []):
        var = anthology.people.PersonName.from_dict(v)
        if var == new:
            return
    variants[i].setdefault("variants", []).append(
        {"first": new.first, "last": new.last}
    )  # don't use to_dict because that adds extra fields


if __name__ == "__main__":
    import argparse
    import logging

    # Set up logging
    logger = logging.getLogger("auto_authors")
    handler = logging.StreamHandler()
    handler.setFormatter(logging.Formatter("%(levelname)s:%(location)s %(message)s"))
    location = ""

    def filter(r):
        r.location = location
        return True

    handler.addFilter(filter)
    logger.setLevel(logging.INFO)
    # logger.setLevel(logging.DEBUG)
    logger.addHandler(handler)
    logger.propagate = False

    scriptdir = os.path.dirname(os.path.abspath(__file__))
    datadir = os.path.join(scriptdir, "..", "data")

    ap = argparse.ArgumentParser(
        description="Try to automatically restore accents in author names."
    )
    ap.add_argument("infiles", metavar="xml", nargs="*", help="input XML files")
    ap.add_argument(
        "-a",
        "--all",
        action="store_true",
        help="process all XML files in input directory",
    )
    ap.add_argument("-p", "--pdfdir", metavar="dir", help="directory with PDFs")
    ap.add_argument(
        "-i", "--indir", metavar="dir", help="input directory", default=datadir
    )
    ap.add_argument(
        "-o", "--outdir", metavar="dir", help="output directory", required=True
    )
    args = ap.parse_args()

    if args.all and args.infiles:
        ap.print_usage(sys.stderr)
        sys.exit("error: the -a and infiles options cannot be used together")
    elif args.all:
        infiles = sorted(glob.glob(os.path.join(datadir, "xml", "*.xml")))
    elif args.infiles:
        infiles = args.infiles
    else:
        ap.print_usage(sys.stderr)
        sys.exit("error: either the -a or infiles option is required")

    anth = anthology.Anthology(importdir=datadir)
    os.makedirs(os.path.join(args.outdir, "data", "xml"), exist_ok=True)
    os.makedirs(os.path.join(args.outdir, "data", "yaml"), exist_ok=True)

    # Although Anthology already read in name_variants.yaml,
    # we read it here so we can modify it
    variants = yaml.safe_load(open(os.path.join(datadir, "yaml", "name_variants.yaml")))

    for infile in infiles:
        tree = etree.parse(infile)
        volume = tree.getroot()
        if not volume.tail:
            volume.tail = "\n"
        for paper in volume.findall("paper"):
            paperid = "{}-{}".format(volume.attrib["id"], paper.attrib["id"])
            location = paperid + ":"

            url = guess_url(paper)
            if url is None:
                continue
            content = get_url(url)
            index = scrape_authors(content)
            if index is None:
                continue

            for authornode in paper.xpath("./author|./editor"):
                firstnode = authornode.find("first")
                lastnode = authornode.find("last")
                if firstnode is None or firstnode.text is None:
                    first = ""
                else:
                    first = firstnode.text
                last = lastnode.text or ""

                location = "{} {} {} {}:".format(paperid, authornode.tag, first, last)

                newfirst = change(index, first)
                newlast = change(index, last)

                person = anth.people.resolve_name(
                    anthology.people.PersonName(first, last)
                )["id"]
                newperson = anth.people.resolve_name(
                    anthology.people.PersonName(newfirst, newlast)
                )["id"]

                merge_people(
                    variants,
                    anth.people.get_canonical_name(person),
                    anth.people.get_canonical_name(newperson),
                )

                if firstnode is not None:
                    firstnode.text = newfirst
                lastnode.text = newlast

        outfile = os.path.join(args.outdir, "data", "xml", os.path.basename(infile))
        tree.write(outfile, xml_declaration=True, encoding="UTF-8")

    variants.sort(key=lambda v: (v["canonical"]["last"], v["canonical"]["first"]))

    with open(
        os.path.join(args.outdir, "data", "yaml", "name_variants.yaml"), "w"
    ) as outfile:
        outfile.write(yaml.dump(variants, allow_unicode=True, default_flow_style=None))


# bin/auto_name_variants.py

import lxml.etree as etree
import sys
import re
import collections
import operator
import unicodedata
import yaml

threshold = 1
print_singletons = False
print_papers = True


def text(node):
    """Extract text from an XML node."""
    if node is None:
        return ""
    s = "".join(node.itertext())
    return " ".join(s.split())


def normalize(s):
    # Split before caps
    s = re.sub(r"([a-z])([A-Z])", r"\1 \2", s)

    # Split on punctuation
    s = "".join(" " if unicodedata.category(c).startswith("P") else c for c in s)

    # Remove accents
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if unicodedata.category(c) != "Mn")

    # Lowercase
    s = s.lower()

    s = " ".join(s.split())

    return s


def distance(x, y):
    """Modified Levenshtein distance, with costs for deleting a suffix or
    a whole word.  Note that this makes the distance asymmetric. This
    is because we don't want low-cost word substitutions.
    """

    c_insert = 1
    c_delete = 1
    c_subst = 1
    c_delete_suffix = 1
    c_delete_word = 1
    c_insert_space = 1
    c_delete_space = 1

    x = x.split() + [""]
    y = y.split() + [""]
    d = collections.defaultdict(lambda: float("inf"))
    d[0, 0, 0, 0] = 0
    for i in range(len(x)):
        for ii in range(len(x[i]) + 1):
            for j in range(len(y)):
                for jj in range(len(y[j]) + 1):
                    if ii > 0 and jj > 0:
                        if x[i][ii - 1] == y[j][jj - 1]:
                            d[i, ii, j, jj] = min(
                                d[i, ii, j, jj], d[i, ii - 1, j, jj - 1]
                            )  # no change
                        else:
                            d[i, ii, j, jj] = min(
                                d[i, ii, j, jj], d[i, ii - 1, j, jj - 1] + c_subst
                            )
                    if ii > 0:
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj], d[i, ii - 1, j, jj] + c_delete
                        )
                    if jj > 0:
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj], d[i, ii, j, jj - 1] + c_insert
                        )

                    if i > 0 and ii == 0 and j > 0 and jj == 0:
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj], d[i - 1, len(x[i - 1]), j - 1, len(y[j - 1])]
                        )  # no change (space)
                        for kk in range(1, len(x[i - 1])):
                            d[i, ii, j, jj] = min(
                                d[i, ii, j, jj],
                                d[i - 1, kk, j - 1, len(y[j - 1])] + c_delete_suffix,
                            )

                    if i > 0 and ii == 0:
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj], d[i - 1, 0, j, jj] + c_delete_word
                        )
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj],
                            d[i - 1, len(x[i - 1]), j, jj] + c_delete_space,
                        )
                    if j > 0 and jj == 0:
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj], d[i, ii, j - 1, 0] + len(y[j - 1])
                        )
                        d[i, ii, j, jj] = min(
                            d[i, ii, j, jj],
                            d[i, ii, j - 1, len(y[j - 1])] + c_insert_space,
                        )

    return d[len(x) - 1, 0, len(y) - 1, 0]


# Simple disjoint-set data structure


def union(p, x, y):
    if x not in p:
        p[x] = None
    if y not in p:
        p[y] = None
    rx = root(p, x)
    ry = root(p, y)
    if rx != ry:
        p[ry] = x  # arbitrary


def root(p, x):
    r = x
    while p[r] is not None:
        r = p[r]
    while x != r:
        temp = p[x]
        p[x] = r
        x = temp
    return r


if __name__ == "__main__":

    keys = collections.defaultdict(list)
    parent = {}
    papers = collections.defaultdict(list)
    counts = collections.Counter()

    # The idea is to generate a list of all possible rotations of all
    # names, e.g., Aravind K. Joshi becomes "aravind k joshi", "k
    # joshi aravind", "joshi aravind k". Then we sort the list and
    # compute edit distances between consecutive names on the list,
    # merging pairs whose distance is below a certain threshold.

    for filename in sys.argv[1:]:
        volume = etree.parse(filename).getroot()
        for paper in volume.findall(".//paper"):
            for person in paper.xpath("./author|./editor"):
                first = text(person.find("first"))
                last = text(person.find("last"))
                counts[first, last] += 1
                papers[first, last].append(
                    "{}-{}".format(volume.attrib["id"], paper.attrib["id"])
                )

                # Make a list of all the normalized name parts
                full = first + " " + last
                full = normalize(full)
                full = full.split()

                # Index the name by all possible rotations of the parts
                for i in range(len(full)):
                    rotated = " ".join(full[i:] + full[:i])
                    keys[rotated].append((first, last))

    for names in keys.values():
        for name in names:
            union(parent, name, names[0])

    # Just measure distance between consecutive names

    prevkey = None
    for key, names in sorted(keys.items()):
        if prevkey is not None:
            d = min(distance(prevkey, key), distance(key, prevkey))
        else:
            d = float("inf")

        if d <= threshold:
            union(parent, names[0], keys[prevkey][0])
        if prevkey is not None:
            print(
                "{}\t{} ({})\t{} ({})".format(
                    d, " ".join(keys[prevkey][0]), prevkey, " ".join(names[0]), key
                ),
                file=sys.stderr,
            )

        prevkey = key

    clusters = collections.defaultdict(list)
    for name in parent:
        clusters[root(parent, name)].append(name)

    # Re-choose canonical names, based on frequency (and then length in case of a tie)

    newclusters = {}
    for cluster, names in clusters.items():
        cname = max(names, key=lambda n: (counts[n], len(n[0]) + len(n[1])))
        newclusters[cname] = names
    clusters = newclusters

    def makename(first, last):
        d = {"first": first, "last": last}
        if print_papers:
            p = sorted(papers[first, last])
            if len(p) > 5:
                p = p[:5]
                p.append("...")
            d["papers"] = " ".join(p)
        return d

    doc = []
    for (cfirst, clast), names in sorted(
        clusters.items(), key=lambda i: (normalize(i[0][1]), normalize(i[0][0]))
    ):
        if len(names) > 1 or print_singletons:
            person = {}
            person["canonical"] = makename(cfirst, clast)
            doc.append(person)
            if len(names) > 1:
                person["variants"] = []
                for first, last in names:
                    if (first, last) == (cfirst, clast):
                        continue
                    person["variants"].append(makename(first, last))

    print(yaml.dump(doc, allow_unicode=True), end="")


# bin/change_authors.py

#!/usr/bin/env python3

"""Apply changes to author names.

usage: change_authors.py -o <out-dir> <change-file>

Reads a list of changes (produced, e.g., by author_case.py)
in the following format:

paperid \t role \t oldfirst || oldlast \t newfirst || newlast

For example:

Z99-9999 \t author \t ARAVIND K. || JOSHI \t Aravind K. || Joshi

means, for paper Z99-999, change author name "ARAVIND K. JOSHI" to
"Aravind K. Joshi".

The script will apply these changes to all of the .xml files given on
the command line and write modified copies in <out-dir>/data/xml.

It will also try to modify data/yaml/name_variants.yaml and write it
to <out-dir>/data/yaml/name_variants.yaml. However, sometimes this is
not possible, so it will ask you to manually merge two entries.
"""

import sys
import os.path
import glob
import anthology
import yaml, yamlfix
import lxml.etree as etree
import collections
import argparse
import logging


def merge_people(variants, can1, can2):
    if can1 == can2:
        return
    # This is really inefficient, but it doesn't actually happen that much
    i1 = i2 = None
    for i, d in enumerate(variants):
        can = anthology.people.PersonName.from_dict(d["canonical"])
        if can == can1:
            i1 = i
        if can == can2:
            i2 = i
    if i1 is not None and i2 is not None:
        logging.error(
            "Please manually merge '{}' and '{}' in name_variants.yaml".format(can1, can2)
        )
        return
    elif i1 is not None:
        i = i1
        new = can2
    elif i2 is not None:
        i = i2
        new = can1
    else:
        # choose can2 to be canonical, since it is the new and hopefully more correct name.
        variants.append({"canonical": {"first": can2.first, "last": can2.last}})
        i = len(variants) - 1
        new = can1
    for v in variants[i].get("variants", []):
        var = anthology.people.PersonName.from_dict(v)
        if var == new:
            return
    variants[i].setdefault("variants", []).append(
        {"first": new.first, "last": new.last}
    )  # don't use to_dict because that adds extra fields


if __name__ == "__main__":
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    datadir = os.path.join(scriptdir, "..", "data")

    logging.basicConfig(level=logging.INFO)

    ap = argparse.ArgumentParser(description="Apply changes to author names.")
    ap.add_argument("changefile", help="list of changes")
    ap.add_argument(
        "-o", "--outdir", metavar="dir", help="output directory", required=True
    )
    args = ap.parse_args()

    changes = {}
    for line in open(args.changefile):
        paperid, role, oldname, newname = line.rstrip().split('\t')
        oldfirst, oldlast = oldname.split(' || ')
        newfirst, newlast = newname.split(' || ')
        changes[paperid, oldfirst, oldlast] = newfirst, newlast

    anth = anthology.Anthology(importdir=datadir)
    variants = yaml.safe_load(open(os.path.join(datadir, "yaml", "name_variants.yaml")))
    infiles = sorted(glob.glob(os.path.join(datadir, "xml", "*.xml")))

    os.makedirs(os.path.join(args.outdir, "data", "xml"), exist_ok=True)
    os.makedirs(os.path.join(args.outdir, "data", "yaml"), exist_ok=True)

    oldnames = collections.defaultdict(list)

    for infile in infiles:
        tree = etree.parse(infile)
        root = tree.getroot()
        if not root.tail:
            root.tail = "\n"
        for volume in root.findall("volume"):
            for paper in volume.findall("paper"):
                paperid = anthology.utils.build_anthology_id(
                    root.attrib["id"], volume.attrib["id"], paper.attrib["id"]
                )
                for authornode in paper.xpath("./author|./editor"):
                    firstnode = authornode.find("first")
                    lastnode = authornode.find("last")
                    if firstnode is None or firstnode.text is None:
                        oldfirst = ""
                    else:
                        oldfirst = firstnode.text
                    oldlast = lastnode.text or ""
                    try:
                        newfirst, newlast = changes[paperid, oldfirst, oldlast]
                    except KeyError:
                        continue

                    # Update variants
                    oldperson = anth.people.resolve_name(
                        anthology.people.PersonName(oldfirst, oldlast)
                    )["id"]
                    newperson = anth.people.resolve_name(
                        anthology.people.PersonName(newfirst, newlast)
                    )["id"]
                    merge_people(
                        variants,
                        anth.people.get_canonical_name(oldperson),
                        anth.people.get_canonical_name(newperson),
                    )

                    # Update XML file
                    if newfirst != "":
                        if firstnode is None:
                            firstnode = etree.SubElement(authornode, 'first')
                        firstnode.text = newfirst
                    else:
                        if firstnode is not None:
                            authornode.remove(firstnode)
                    lastnode.text = newlast

                    oldnames[anthology.people.PersonName(oldfirst, oldlast)].append(
                        paperid
                    )

        outfile = os.path.join(args.outdir, "data", "xml", os.path.basename(infile))
        tree.write(outfile, xml_declaration=True, encoding="UTF-8")

    # If a name variant is no longer used, delete it
    deleted_names = set()
    for name, papers in oldnames.items():
        if set(papers) == set(anth.pindex.name_to_papers[name][False]):
            deleted_names.add(name)

    newvariants = []
    for d in variants:
        name = anthology.people.PersonName.from_dict(d["canonical"])
        if name in deleted_names:
            logging.error(
                "canonical name '{}' is no longer used; please delete manually".format(
                    name
                )
            )
        for var in list(d.get("variants", [])):
            name = anthology.people.PersonName.from_dict(var)
            if name in deleted_names:
                logging.info("variant name '{}' is no longer used; deleting".format(name))
                d["variants"].remove(var)
        if "variants" in d and len(d["variants"]) == 0:
            del d["variants"]
        if list(d.keys()) == ["canonical"]:
            continue
        newvariants.append(d)
    variants = newvariants

    variants.sort(key=lambda v: (v["canonical"]["last"], v["canonical"]["first"]))

    with open(
        os.path.join(args.outdir, "data", "yaml", "name_variants.yaml"), "w"
    ) as outfile:
        outfile.write(yaml.dump(variants, allow_unicode=True, default_flow_style=None))


# bin/clean_name_variants.py

import yaml, yamlfix
import sys
import anthology
import logging
import os.path

scriptdir = os.path.dirname(os.path.abspath(__file__))
datadir = os.path.join(scriptdir, "..", "data")

anth = anthology.Anthology(importdir=datadir)

variants = yaml.safe_load(open(os.path.join(datadir, "yaml", "name_variants.yaml")))

newvariants = []
for d in variants:
    name = anthology.people.PersonName.from_dict(d["canonical"])
    if len(anth.people.name_to_papers[name][False]) == 0:
        logging.info("unused canonical name: {}".format(name))
    for var in list(d.get("variants", [])):
        name = anthology.people.PersonName.from_dict(var)
        if len(anth.people.name_to_papers[name][False]) == 0:
            logging.info("pruning unused variant: {}".format(name))
            d["variants"].remove(var)
    if "variants" in d and len(d["variants"]) == 0:
        del d["variants"]
    if list(d.keys()) == ["canonical"]:
        continue
    newvariants.append(d)

newvariants.sort(key=lambda v: (v["canonical"]["last"], v["canonical"]["first"]))

sys.stdout.write(yaml.dump(newvariants, allow_unicode=True, default_flow_style=None))


# bin/consolidate_urls.py

#!/usr/bin/env python3

"""
Used to clean up the URL situation:

- 'href' tags on papers are moved to <url> tags
- <href> tags are turned intot <url> tags
- 'https?:(www).aclweb.org/anthology/' is stripped from <url> tags, leaving just the relative ACL ID

Usage: consolidate_urls.py <infilename> <outfilename>

"""

import lxml.etree as etree
import re
import sys

from repair_url import test_url, get_anth_url

filename = sys.argv[1]
outfilename = sys.argv[2]
tree = etree.parse(filename)
volume = tree.getroot()
volume_id = volume.attrib["id"]

# Build list of volumes observed
volumes = set()

for paper in volume.findall("paper"):
    href = None

    if volume_id.startswith("W"):
        volumes.add(paper.attrib["id"][0:2])
    else:
        volumes.add(paper.attrib["id"][0:1])

    paper_id = int(paper.attrib["id"])
    acl_id = "{}-{}".format(volume_id, paper_id)
    if "href" in paper.attrib:
        href = paper.attrib["href"]
        print("{}: removing href attribute {}".format(acl_id, href), file=sys.stderr)
        del paper.attrib["href"]

    if href and paper.find("href"):
        print(
            f'{acl_id}: WARNING: found "href" attribute AND <href> tag', file=sys.stderr
        )
        sys.exit(1)

    if paper.find("href"):
        assert len(href) == 0
        if not test_url(href.text):
            href = paper.find("href").text
            print(f"{acl_id}: removing href element: {text}", file=sys.stderr)
            paper.remove(paper.find("href"))

    url = paper.find("url")
    if href and url:
        print(
            f'{paper_id}: WARNING: found "href" attribute AND <url> tag', file=sys.stderr
        )
        sys.exit(1)

    if url is not None:
        anth_url = url.text
    elif href and href.startswith("http"):
        anth_url = href
    else:
        anth_url = "{}-{:04d}".format(volume_id, paper_id)

    # Create the node if missing
    if url is None:
        print(f"{acl_id}: inserting new url element", file=sys.stderr)
        url = etree.Element("url")
        url.tail = "\n    "
        paper.append(url)

    new_anth_url = re.sub(r"^https?://(www\.)?aclweb.org/anthology/", "", anth_url)
    if new_anth_url != anth_url:
        print(f"{acl_id}: rewriting url: {anth_url} -> {new_anth_url}", file=sys.stderr)
        anth_url = new_anth_url

    url.text = anth_url

# Now look for volumes -- (postponed until after hierarchical change)
# for id_ in sorted(volumes):
#     url = etree.Element('url')
#     url.text = get_anth_url(volume_id, int(id_), width=len(id_))
#     if test_url(url.text):
#         print("{}: inserting volume URL: {}".format(volume_id, url.text), file=sys.stderr)
#         url.tail = '\n  '
#         volume.insert(0, url)

# https://stackoverflow.com/a/33956544
# def indent(elem, level=0):
#     i = "\n" + level*"  "
#     if len(elem):
#         if not elem.text or not elem.text.strip():
#             elem.text = i + "  "
#         if not elem.tail or not elem.tail.strip():
#             elem.tail = i
#         for elem in elem:
#             indent(elem, level+1)
#         if not elem.tail or not elem.tail.strip():
#             elem.tail = i
#     else:
#         if level and (not elem.tail or not elem.tail.strip()):
#             elem.tail = i

# indent(volume)

tree.write(outfilename, encoding="UTF-8", xml_declaration=True, with_tail=True)


# bin/create_bibtex.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: create_bibtex.py [--importdir=DIR] [--exportdir=DIR] [-c] [--debug]

Creates .bib files for all papers in the Hugo directory.

Options:
  --importdir=DIR          Directory to import XML files from. [default: {scriptdir}/../data/]
  --exportdir=DIR          Directory to write exported files to.   [default: {scriptdir}/../build/data-export/]
  --debug                  Output debug-level log messages.
  -c, --clean              Delete existing files in target directory before generation.
  -h, --help               Display this helpful text.
"""

from docopt import docopt
from lxml import etree
from tqdm import tqdm
import gzip
import logging as log
import io
import os

from anthology import Anthology
from anthology.utils import SeverityTracker, deconstruct_anthology_id, infer_year
from create_hugo_pages import check_directory
from operator import itemgetter


def volume_sorter(volume_tuple):
    """
    Extracts the year so that we can sort by the year and then
    the collection ID.
    """
    volume_id = volume_tuple[0]
    collection_id, year, _ = deconstruct_anthology_id(volume_id)
    year = infer_year(collection_id)
    return year, volume_id


def create_bibtex(anthology, trgdir, clean=False):
    """Creates .bib files for all papers."""
    if not check_directory("{}/papers".format(trgdir), clean=clean):
        return
    if not check_directory("{}/volumes".format(trgdir), clean=clean):
        return

    log.info("Creating BibTeX files for all papers...")
    with open(
        "{}/anthology.bib".format(trgdir), "wt", encoding="utf-8"
    ) as file_anthology_raw, gzip.open(
        "{}/anthology.bib.gz".format(trgdir), "wt", encoding="utf-8"
    ) as file_anthology, gzip.open(
        "{}/anthology+abstracts.bib.gz".format(trgdir), "wt", encoding="utf-8"
    ) as file_anthology_with_abstracts:
        for volume_id, volume in tqdm(
            sorted(anthology.volumes.items(), key=volume_sorter, reverse=True)
        ):
            volume_dir = trgdir
            if not os.path.exists(volume_dir):
                os.makedirs(volume_dir)
            with open("{}/volumes/{}.bib".format(trgdir, volume_id), "w") as file_volume:
                for paper in volume:
                    with open(
                        "{}/{}.bib".format(volume_dir, paper.full_id), "w"
                    ) as file_paper:
                        contents = paper.as_bibtex()
                        print(contents, file=file_paper)
                        print(contents, file=file_anthology_with_abstracts)

                        concise_contents = paper.as_bibtex(concise=True)
                        print(concise_contents, file=file_volume)
                        print(concise_contents, file=file_anthology)
                        print(concise_contents, file=file_anthology_raw)


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )
    if "{scriptdir}" in args["--exportdir"]:
        args["--exportdir"] = os.path.abspath(
            args["--exportdir"].format(scriptdir=scriptdir)
        )

    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)

    anthology = Anthology(importdir=args["--importdir"], fast_load=True)
    create_bibtex(anthology, args["--exportdir"], clean=args["--clean"])

    if tracker.highest >= log.ERROR:
        exit(1)


# bin/create_hugo_pages.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: create_hugo_pages.py [--dir=DIR] [-c] [--debug]

Creates page stubs for the full anthology based on the YAML data files.

This script can only be run after create_hugo_yaml.py!

Options:
  --dir=DIR                Hugo project directory. [default: {scriptdir}/../build/]
  --debug                  Output debug-level log messages.
  -c, --clean              Delete existing files in target directory before generation.
  -h, --help               Display this helpful text.
"""

from docopt import docopt
from glob import glob
from slugify import slugify
from tqdm import tqdm
import logging as log
import os
import shutil
import yaml

try:
    from yaml import CLoader as Loader
except ImportError:
    log.info("Can't load yaml C bindings, reverting to slow pure Python version")
    from yaml import Loader

from anthology.utils import SeverityTracker


def check_directory(cdir, clean=False):
    if not os.path.isdir(cdir) and not os.path.exists(cdir):
        os.mkdir(cdir)
        return True
    entries = os.listdir(cdir)
    if "_index.md" in entries:
        entries.remove("_index.md")
    if entries and not clean:
        log.critical("Directory already exists and has content files: {}".format(cdir))
        log.info(
            "Call this script with the -c/--clean flag to automatically DELETE existing files"
        )
        return False
    for entry in entries:
        entry = "{}/{}".format(cdir, entry)
        if os.path.isdir(entry):
            shutil.rmtree(entry)
        else:
            os.remove(entry)
    return True


def create_papers(srcdir, clean=False):
    """Creates page stubs for all papers in the Anthology."""
    log.info("Creating stubs for papers...")
    if not check_directory("{}/content/papers".format(srcdir), clean=clean):
        return

    # Go through all paper volumes
    for yamlfile in tqdm(glob("{}/data/papers/*.yaml".format(srcdir))):
        log.debug("Processing {}".format(yamlfile))
        with open(yamlfile, "r") as f:
            data = yaml.load(f, Loader=Loader)
        # Create a paper stub for each entry in the volume
        for anthology_id, entry in data.items():
            paper_dir = "{}/content/papers/{}".format(srcdir, anthology_id.split("-")[0])
            if not os.path.exists(paper_dir):
                os.makedirs(paper_dir)
            with open("{}/{}.md".format(paper_dir, anthology_id), "w") as f:
                print("---", file=f)
                yaml.dump(
                    {"anthology_id": anthology_id, "title": entry["title"]},
                    default_flow_style=False,
                    stream=f,
                )
                print("---", file=f)


def create_volumes(srcdir, clean=False):
    """Creates page stubs for all proceedings volumes in the Anthology."""
    log.info("Creating stubs for volumes...")
    if not check_directory("{}/content/volumes".format(srcdir), clean=clean):
        return

    yamlfile = "{}/data/volumes.yaml".format(srcdir)
    log.debug("Processing {}".format(yamlfile))
    with open(yamlfile, "r") as f:
        data = yaml.load(f, Loader=Loader)
    # Create a paper stub for each proceedings volume
    for anthology_id, entry in data.items():
        with open("{}/content/volumes/{}.md".format(srcdir, anthology_id), "w") as f:
            print("---", file=f)
            yaml.dump(
                {
                    "anthology_id": anthology_id,
                    "title": entry["title"],
                },
                default_flow_style=False,
                stream=f,
            )
            print("---", file=f)

    return data


def create_people(srcdir, clean=False):
    """Creates page stubs for all authors/editors in the Anthology."""
    log.info("Creating stubs for people...")
    if not check_directory("{}/content/people".format(srcdir), clean=clean):
        return

    for yamlfile in tqdm(glob("{}/data/people/*.yaml".format(srcdir))):
        log.debug("Processing {}".format(yamlfile))
        with open(yamlfile, "r") as f:
            data = yaml.load(f, Loader=Loader)
        # Create a page stub for each person
        for name, entry in data.items():
            person_dir = "{}/content/people/{}".format(srcdir, name[0])
            if not os.path.exists(person_dir):
                os.makedirs(person_dir)
            yaml_data = {"name": name, "title": entry["full"], "lastname": entry["last"]}
            with open("{}/{}.md".format(person_dir, name), "w") as f:
                print("---", file=f)
                # "lastname" is dumped to allow sorting by it in Hugo
                yaml.dump(yaml_data, default_flow_style=False, stream=f)
                print("---", file=f)

    return data


def create_venues_and_events(srcdir, clean=False):
    """Creates page stubs for all venues and events in the Anthology."""
    yamlfile = "{}/data/venues.yaml".format(srcdir)
    log.debug("Processing {}".format(yamlfile))
    with open(yamlfile, "r") as f:
        data = yaml.load(f, Loader=Loader)

    log.info("Creating stubs for venues...")
    if not check_directory("{}/content/venues".format(srcdir), clean=clean):
        return
    # Create a paper stub for each venue (e.g. ACL)
    for venue, venue_data in data.items():
        venue_str = venue_data["slug"]
        with open("{}/content/venues/{}.md".format(srcdir, venue_str), "w") as f:
            print("---", file=f)
            yaml_data = {"venue": venue, "title": venue_data["name"]}
            yaml.dump(yaml_data, default_flow_style=False, stream=f)
            print("---", file=f)

    log.info("Creating stubs for events...")
    if not check_directory("{}/content/events".format(srcdir), clean=clean):
        return
    # Create a paper stub for each event (= venue + year, e.g. ACL 2018)
    for venue, venue_data in data.items():
        venue_str = venue_data["slug"]
        for year in venue_data["volumes_by_year"]:
            with open(
                "{}/content/events/{}-{}.md".format(srcdir, venue_str, year), "w"
            ) as f:
                print("---", file=f)
                yaml_data = {
                    "venue": venue,
                    "year": year,
                    "title": "{} ({})".format(venue_data["name"], year),
                }
                yaml.dump(yaml_data, default_flow_style=False, stream=f)
                print("---", file=f)


def create_sigs(srcdir, clean=False):
    """Creates page stubs for all SIGs in the Anthology."""
    yamlfile = "{}/data/sigs.yaml".format(srcdir)
    log.debug("Processing {}".format(yamlfile))
    with open(yamlfile, "r") as f:
        data = yaml.load(f, Loader=Loader)

    log.info("Creating stubs for SIGs...")
    if not check_directory("{}/content/sigs".format(srcdir), clean=clean):
        return
    # Create a paper stub for each SIGS (e.g. SIGMORPHON)
    for sig, sig_data in data.items():
        sig_str = sig_data["slug"]
        with open("{}/content/sigs/{}.md".format(srcdir, sig_str), "w") as f:
            print("---", file=f)
            yaml.dump(
                {
                    "acronym": sig,
                    "short_acronym": sig[3:] if sig.startswith("SIG") else sig,
                    "title": sig_data["name"],
                },
                default_flow_style=False,
                stream=f,
            )
            print("---", file=f)


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--dir"]:
        args["--dir"] = args["--dir"].format(scriptdir=scriptdir)
    dir_ = os.path.abspath(args["--dir"])

    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)

    create_papers(dir_, clean=args["--clean"])
    create_volumes(dir_, clean=args["--clean"])
    create_people(dir_, clean=args["--clean"])
    create_venues_and_events(dir_, clean=args["--clean"])
    create_sigs(dir_, clean=args["--clean"])

    if tracker.highest >= log.ERROR:
        exit(1)


# bin/create_hugo_yaml.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: create_hugo_yaml.py [--importdir=DIR] [--exportdir=DIR] [-c] [--debug] [--dry-run]

Creates YAML files containing all necessary Anthology data for the static website generator.

Options:
  --importdir=DIR          Directory to import XML files from. [default: {scriptdir}/../data/]
  --exportdir=DIR          Directory to write YAML files to.   [default: {scriptdir}/../build/data/]
  --debug                  Output debug-level log messages.
  -c, --clean              Delete existing files in target directory before generation.
  -n, --dry-run            Do not write YAML files (useful for debugging).
  -h, --help               Display this helpful text.
"""

from docopt import docopt
from collections import defaultdict
from tqdm import tqdm
import logging as log
import os
import yaml, yamlfix

try:
    from yaml import CSafeDumper as Dumper
except ImportError:
    from yaml import SafeDumper as Dumper

from anthology import Anthology
from anthology.utils import SeverityTracker, deconstruct_anthology_id, is_newstyle_id
from create_hugo_pages import check_directory


def export_anthology(anthology, outdir, clean=False, dryrun=False):
    # Prepare paper index
    papers = defaultdict(dict)
    citation_styles = {
        "acl": "association-for-computational-linguistics",
        # "apa": "apa-6th-edition",
        # "mla": "modern-language-association-7th-edition",
    }
    for id_, paper in anthology.papers.items():
        log.debug("export_anthology: processing paper '{}'".format(id_))
        data = paper.as_dict()
        data["title_html"] = paper.get_title("html")
        if "xml_title" in data:
            del data["xml_title"]
        if "xml_booktitle" in data:
            data["booktitle_html"] = paper.get_booktitle("html")
            del data["xml_booktitle"]
        if "xml_abstract" in data:
            data["abstract_html"] = paper.get_abstract("html")
            del data["xml_abstract"]
        if "xml_url" in data:
            del data["xml_url"]
        if "author" in data:
            data["author"] = [
                anthology.people.resolve_name(name, id_) for name, id_ in data["author"]
            ]
        if "editor" in data:
            data["editor"] = [
                anthology.people.resolve_name(name, id_) for name, id_ in data["editor"]
            ]
        for abbrev, style in citation_styles.items():
            data[f"citation_{abbrev}"] = paper.as_citation_html(style)
        papers[paper.collection_id][paper.full_id] = data

    # Prepare people index
    people = defaultdict(dict)
    for id_ in anthology.people.personids():
        name = anthology.people.get_canonical_name(id_)
        log.debug("export_anthology: processing person '{}'".format(repr(name)))
        data = name.as_dict()
        data["slug"] = id_
        if id_ in anthology.people.comments:
            data["comment"] = anthology.people.comments[id_]
        if id_ in anthology.people.similar:
            data["similar"] = sorted(anthology.people.similar[id_])
        data["papers"] = sorted(
            anthology.people.get_papers(id_),
            key=lambda p: anthology.papers.get(p).get("year"),
            reverse=True,
        )
        data["coauthors"] = sorted(
            [[co_id, count] for (co_id, count) in anthology.people.get_coauthors(id_)],
            key=lambda p: p[1],
            reverse=True,
        )
        data["venues"] = sorted(
            [
                [venue, count]
                for (venue, count) in anthology.people.get_venues(
                    anthology.venues, id_
                ).items()
            ],
            key=lambda p: p[1],
            reverse=True,
        )
        variants = [
            n
            for n in anthology.people.get_used_names(id_)
            if n.first != name.first or n.last != name.last
        ]
        if len(variants) > 0:
            data["variant_entries"] = [name.as_dict() for name in sorted(variants)]
        people[id_[0]][id_] = data

    # Prepare volume index
    volumes = {}
    for id_, volume in anthology.volumes.items():
        log.debug("export_anthology: processing volume '{}'".format(id_))
        data = volume.as_dict()
        data["title_html"] = volume.get_title("html")
        del data["xml_booktitle"]
        if "xml_abstract" in data:
            del data["xml_abstract"]
        if "xml_url" in data:
            del data["xml_url"]
        data["has_abstracts"] = volume.has_abstracts
        data["papers"] = volume.paper_ids
        if "author" in data:
            data["author"] = [
                anthology.people.resolve_name(name, id_) for name, id_ in data["author"]
            ]
        if "editor" in data:
            data["editor"] = [
                anthology.people.resolve_name(name, id_) for name, id_ in data["editor"]
            ]
        volumes[volume.full_id] = data

    class SortedVolume:
        """Keys for sorting volumes so they appear in a more reasonable order.
        Takes the parent venue being sorted under, along with its letter,
        and the Anthology ID of the current volume. For example, LREC 2020
        has the following joint events, which get sorted in the following manner:

        ['2020.lrec-1', '2020.aespen-1', '2020.ai4hi-1',
        '2020.bucc-1', '2020.calcs-1', '2020.cllrd-1', '2020.clssts-1',
        '2020.cmlc-1', '2020.computerm-1', '2020.framenet-1', '2020.gamnlp-1',
        '2020.globalex-1', '2020.isa-1', '2020.iwltp-1',
        '2020.ldl-1', '2020.lincr-1', '2020.lr4sshoc-1', '2020.lt4gov-1',
        '2020.lt4hala-1 ', '2020.multilingualbio-1', '2020.onion-1',
        '2020.osact-1', '2020.parlaclarin-1', '2020.rail-1', '2020.readi-1',
        '2020.restup-1', '2020.sltu-1 ', '2020.stoc-1', '2020.trac-1',
        '2020.wac-1', '2020.wildre-1']
        """

        def __init__(self, acronym, letter, anth_id):
            self.parent_venue = acronym.lower()
            self.anth_id = anth_id

            collection_id, self.volume_id, _ = deconstruct_anthology_id(anth_id)
            if is_newstyle_id(collection_id):
                self.venue = collection_id.split(".")[1]
                self.is_parent_venue = self.venue == self.parent_venue
            else:
                self.venue = collection_id[0]
                self.is_parent_venue = self.venue == letter

        def __str__(self):
            return self.anth_id

        def __eq__(self, other):
            """We define equivalence at the venue (not volume) level in order
            to preserve the sort order found in the XML"""
            return self.venue == other.venue

        def __lt__(self, other):
            """First parent volumes, then sort by venue name"""
            if self.is_parent_venue == other.is_parent_venue:
                return self.venue < other.venue
            return self.is_parent_venue and not other.is_parent_venue

    # Prepare venue index
    venues = {}
    for acronym, data in anthology.venues.items():
        letter = data.get("oldstyle_letter", "W")
        data = data.copy()
        data["volumes_by_year"] = {
            year: sorted(
                filter(lambda k: volumes[k]["year"] == year, data["volumes"]),
                key=lambda x: SortedVolume(acronym, letter, x),
            )
            for year in sorted(data["years"])
        }
        data["years"] = sorted(list(data["years"]))
        del data["volumes"]
        venues[acronym] = data

    # Prepare SIG index
    sigs = {}
    for acronym, sig in anthology.sigs.items():
        data = {
            "name": sig.name,
            "slug": sig.slug,
            "url": sig.url,
            "volumes_by_year": sig.volumes_by_year,
            "years": sorted([str(year) for year in sig.years]),
        }
        sigs[acronym] = data

    # Dump all
    if not dryrun:
        # Create directories
        for subdir in ("", "papers", "people"):
            target_dir = "{}/{}".format(outdir, subdir)
            if not check_directory(target_dir, clean=clean):
                return

        progress = tqdm(total=len(papers) + len(people) + 7)
        for collection_id, paper_list in papers.items():
            with open("{}/papers/{}.yaml".format(outdir, collection_id), "w") as f:
                yaml.dump(paper_list, Dumper=Dumper, stream=f)
            progress.update()

        with open("{}/volumes.yaml".format(outdir), "w") as f:
            yaml.dump(volumes, Dumper=Dumper, stream=f)
        progress.update(5)

        with open("{}/venues.yaml".format(outdir), "w") as f:
            yaml.dump(venues, Dumper=Dumper, stream=f)
        progress.update()

        with open("{}/sigs.yaml".format(outdir), "w") as f:
            yaml.dump(sigs, Dumper=Dumper, stream=f)
        progress.update()

        for first_letter, people_list in people.items():
            with open("{}/people/{}.yaml".format(outdir, first_letter), "w") as f:
                yaml.dump(people_list, Dumper=Dumper, stream=f)
            progress.update()
        progress.close()


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )
    if "{scriptdir}" in args["--exportdir"]:
        args["--exportdir"] = os.path.abspath(
            args["--exportdir"].format(scriptdir=scriptdir)
        )

    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)

    log.info("Reading the Anthology data...")
    anthology = Anthology(importdir=args["--importdir"])
    log.info("Exporting to YAML...")
    export_anthology(
        anthology, args["--exportdir"], clean=args["--clean"], dryrun=args["--dry-run"]
    )

    if tracker.highest >= log.ERROR:
        exit(1)


# bin/create_mirror.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020-2021 Arne Köhn <arne@chark.eu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: create_mirror.py [--source=SRC] [--to=DIR] [--debug] [--dry-run] [--only-papers] XMLFILE...

Creates YAML files containing all necessary Anthology data for the static website generator.
XMLFILE: data file from the anthology data dir to fetch data for. Use data/xml/*.xml to
fetch everything.

Options:
  --source=SRC               where to fetch the files from [default: https://aclanthology.org]
  --to=DIR                 Directory to write files to [default: {scriptdir}/../build/anthology-files]
  --only-papers            Do not mirror attachments, only papers.
  --debug                  Output debug-level log messages.
  -n, --dry-run            Do not actually download, use with --debug to see what would happen
  -h, --help               Display this helpful text.
"""

import anthology.utils as anthology_utils
from anthology.utils import SeverityTracker
from docopt import docopt
import logging as log
import os
import re
import shutil
import sys
import tempfile

from urllib.request import urlretrieve

from lxml import etree

####  COPY OF .htaccess RULES USED AS GUIDE FOR THE REGULAR EXPRESSIONS BELOW
#############################################################################
# # PDF link, revisions, and errata (P17-1069[v2].pdf loads P/P17/P17-1069[v2].pdf --- with "v2" optional)
# # TODO: decide on a new format for revisions and errata
# RewriteRule ^([A-Za-z])(\d{2})\-(\d{4})([ve]\d+)?\.pdf$ /ANTHOLOGYFILES/pdf/$1/$1$2/$1$2-$3$4.pdf [L,NC]
# RewriteRule ^(\d{4})\.([a-zA-Z\d]+)\-([a-zA-Z\d]+)\.(\d+([ve]\d+)?)\.pdf$ /ANTHOLOGYFILES/pdf/$2/$1.$2-$3.$4.pdf [L,NC]

# # Attachments (e.g., P17-1069.Poster.pdf loads /ANTHOLOGYFILES/attachments/P/P17/P17-1069.Poster.pdf)
# RewriteRule ^attachments/([A-Za-z])(\d{2})\-(\d{4})(\..*)?$ /ANTHOLOGYFILES/attachments/$1/$1$2/$1$2-$3$4 [L,NC]
# RewriteRule ^attachments/(\d{4})\.([a-zA-Z\d]+)\-([a-zA-Z\d]+\.\d+)\.(.*)$ /ANTHOLOGYFILES/attachments/$2/$1.$2-$3.$4 [L,NC]

NEW_ID_RE = re.compile(r"^(\d{4})\.([a-zA-Z\d]+)\-")
OLD_ID_RE = re.compile(r"^([A-Za-z])(\d{2})\-")


def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)


class ACLMirrorer:
    def __init__(self, args) -> None:
        self.hash_mismatches = []
        self.not_downloadable = []
        self.args = args
        self.source = args["--source"]
        self.to = args["--to"]
        self.is_dry_run = args["--dry-run"]
        self.only_papers = args["--only-papers"]

    def download_file(self, fname: str, hash: str, type: str) -> None:
        if type == "pdf":
            remote_url = self.source + "/" + fname
        elif type == "attachment":
            remote_url = self.source + "/attachments/" + fname
        else:
            log.error("unrecognized type: " + type)
            exit(1)
        local_target = ""
        # mkstemp opens the file but we only need the file name, so close it again.
        tmpfd, tmp_target = tempfile.mkstemp(prefix="aclmirrorer_", suffix=".pdf")
        os.close(tmpfd)
        match = NEW_ID_RE.match(fname)
        if match:
            local_target = os.path.join(self.to, type, match.groups()[1], fname)
        else:
            match = OLD_ID_RE.match(fname)
            if match:
                local_target = os.path.join(
                    self.to,
                    type,
                    match.groups()[0],
                    match.groups()[0] + match.groups()[1],
                    fname,
                )
            else:
                log.error("unrecognized format for " + fname)
                exit(1)

        if os.path.exists(local_target):
            existing_hash = anthology_utils.compute_hash_from_file(local_target)
            if existing_hash == hash:
                log.debug(
                    "File {} already up to date, not downloading again".format(
                        local_target
                    )
                )
                return
            else:
                log.debug("File {} changed, redownloading ...".format(local_target))
        else:
            log.debug("Downloading {} from {} ...".format(local_target, remote_url))

        if self.is_dry_run:
            return

        local_path = os.path.dirname(local_target)
        os.makedirs(local_path, exist_ok=True)
        try:
            urlretrieve(remote_url, tmp_target)
        except:
            log.error("could not download " + remote_url)
            self.not_downloadable.append(remote_url)
            os.remove(tmp_target)
            return
        new_hash = anthology_utils.compute_hash_from_file(tmp_target)
        if new_hash == hash:
            # all good, store downloaded file in the proper place
            shutil.move(tmp_target, local_target)
        else:
            log.error(
                "Hash mismatch for file {}, downloaded from {}. was {} should be {}".format(
                    local_target, remote_url, new_hash, hash
                )
            )
            self.hash_mismatches.append(remote_url)
        return

    def download_files(self, xmlfname: str):
        xml = etree.parse(xmlfname)
        proceedings = xml.findall("//volume/meta/url[@hash]")
        frontmatter = xml.findall("//frontmatter/url[@hash]")
        papers = xml.findall("//paper/url[@hash]")
        attachments = xml.findall("//paper/attachment[@hash]")
        revisions = xml.findall("//paper/revision[@hash]")
        errata = xml.findall("//paper/erratum[@hash]")
        log.info("processing {} papers from {} ...".format(len(papers), xmlfname))
        for collection in [proceedings, frontmatter, papers, revisions, errata]:
            for entry in collection:
                hash = entry.attrib["hash"]
                # revisions encode the file name in a href attribute instead of the text
                if "href" in entry.attrib:
                    fname = entry.attrib["href"]
                else:
                    fname = entry.text
                self.download_file(fname + ".pdf", hash, "pdf")
        if self.only_papers:
            for entry in attachments:
                hash = entry.attrib["hash"]
                fname = entry.text
                self.download_file(fname, hash, "attachment")


def main():
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--to"]:
        args["--to"] = args["--to"].format(scriptdir=scriptdir)
    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)
    mirrorer = ACLMirrorer(args)
    for f in args["XMLFILE"]:
        log.info("processing {} ...".format(f))
        mirrorer.download_files(f)
    eprint("\nFiles that could not be downloaded")
    eprint("==================================")
    for elem in mirrorer.not_downloadable:
        eprint(elem)
    eprint("\n\nFiles with checksum mismatch")
    eprint("============================")
    for elem in mirrorer.hash_mismatches:
        eprint(elem)


if __name__ == "__main__":
    main()


# bin/diff_xml.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 David Wei Chiang <dchiang@nd.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import difflib
import lxml.etree as etree
import operator
import copy

unique_attributes = {"id"}
elements = {"volume", "paper", "author", "editor"}
oldcolor = "\033[91m\033[7m"
newcolor = "\033[92m\033[7m"
nocolor = "\033[0m"
max_length = 65  # information in left margin is 14 chars


def opentag(node):
    copy = etree.Element(node.tag)
    copy.text = "\n"
    for a, v in sorted(node.attrib.items()):
        if a in unique_attributes:
            copy.attrib[a] = v
    s = etree.tostring(copy, encoding=str)
    return s.splitlines()[0]


def tostring(node, max_length=None):
    node = copy.deepcopy(node)

    # Strip leading and trailing whitespace
    if node.text is not None:
        node.text = node.text.lstrip()
    if len(node) > 0:
        if node[-1].tail is not None:
            node[-1].tail = node[-1].tail.rstrip()
    else:
        if node.text is not None:
            node.text = node.text.rstrip()

    s = etree.tostring(node, encoding=str, with_tail=False)
    s = " ".join(s.split())
    if max_length is not None:
        return s[:max_length]
    else:
        return s


def diff_strings(a, b, aloc, bloc, max_length):
    sm = difflib.SequenceMatcher(a=a, b=b)
    ops = sm.get_opcodes()

    min_context = 15

    def visit(ai, aj, bi, bj):
        if aj - ai == bj - bi == 0:
            return

        # Widen the windows by min_context
        aa = max(0, ai - min_context)
        az = min(len(a), aj + min_context)
        ba = max(0, bi - min_context)
        bz = min(len(b), bj + min_context)

        if az - aa <= max_length and bz - ba <= max_length:
            # Widen more
            aa = max(0, min(ai - (max_length - (aj - ai)) // 2, len(a) - max_length))
            az = aa + max_length
            ba = max(0, min(bi - (max_length - (bj - bi)) // 2, len(b) - max_length))
            bz = ba + max_length

            # Print the differences all together
            print("{:5d} {:5s} - ".format(aloc, ""), end="")
            for op, ak, al, bk, bl in sm.get_opcodes():
                if ai == ak and bi == bk:  # first piece, print left context
                    print(a[aa:ai], end="")
                if ai <= ak <= al <= aj:
                    if op == "equal":
                        print(a[ak:al], end="")
                    else:
                        print(oldcolor + a[ak:al] + nocolor, end="")
                if al == aj and bl == bj:  # last piece, print right context
                    print(a[aj:az], end="")
            print()

            print("{:5s} {:5d} + ".format("", bloc), end="")
            for op, ak, al, bk, bl in sm.get_opcodes():
                if ai == ak and bi == bk:
                    print(b[ba:bi], end="")
                if ai <= ak <= al <= aj:
                    if op == "equal":
                        print(b[bk:bl], end="")
                    else:
                        print(newcolor + b[bk:bl] + nocolor, end="")
                if al == aj and bl == bj:
                    print(b[bj:bz], end="")
            print()

        else:
            # Try to split the strings in two (same way that difflib does it)
            ak, bk, l = sm.find_longest_match(ai, aj, bi, bj)
            if l > 0:
                visit(ai, ak, bi, bk)
                visit(ak + l, aj, bk + l, bj)
            else:
                # Just print the whole string with ... in the middle
                aij = a[ai:aj]
                if az - aa > max_length:
                    l = (max_length - 3 - (ai - aa) - (az - aj)) // 2
                    if l >= 0:
                        aij = a[ai : ai + l] + "..." + a[aj - l : aj]
                bij = b[bi:bj]
                if bz - ba > max_length:
                    l = (max_length - 3 - (bi - ba) - (bz - bj)) // 2
                    if l >= 0:
                        bij = b[bi : bi + l] + "..." + b[bj - l : bj]

                print(
                    "{:5d} {:5s} - {}".format(
                        aloc, "", a[aa:ai] + oldcolor + aij + nocolor + a[aj:az]
                    )
                )
                print(
                    "{:5s} {:5d} + {}".format(
                        "", bloc, b[ba:bi] + newcolor + bij + nocolor + b[bj:bz]
                    )
                )

    # strip outermost non-differences
    diffs = [op for op in ops if op[0] != "equal"]
    _, ai, _, bi, _ = diffs[0]
    _, _, aj, _, bj = diffs[-1]
    visit(ai, aj, bi, bj)


def unified_diff(oldtree, newtree):
    def visit(oldnode, newnode):
        # Usually oldnode and newnode have the same tag, unless they are
        # both the root

        if oldnode.tag not in elements and newnode.tag not in elements:
            # Just compare the nodes as strings
            oldstring = tostring(oldnode)
            newstring = tostring(newnode)
            if oldstring != newstring:
                diff_strings(
                    oldstring,
                    newstring,
                    oldnode.sourceline,
                    newnode.sourceline,
                    max_length,
                )

        else:
            if (oldnode.text or "").strip():
                print(
                    "warning: text at start of element at {}:{}".format(
                        args.oldfile, oldnode.sourceline
                    )
                )
            if (newnode.text or "").strip():
                print(
                    "warning: text at start of element at {}:{}".format(
                        args.newfile, newnode.sourceline
                    )
                )
            for c in oldnode:
                if (c.tail or "").strip():
                    print(
                        "warning: text after element at {}:{}".format(
                            args.oldfile, c.sourceline
                        )
                    )
            for c in newnode:
                if (c.tail or "").strip():
                    print(
                        "warning: text after element at {}:{}".format(
                            args.newfile, c.sourceline
                        )
                    )

            oldchildren = sorted(oldnode, key=opentag)
            newchildren = sorted(newnode, key=opentag)

            for op, oi, oj, ni, nj in difflib.SequenceMatcher(
                a=list(map(opentag, oldchildren)), b=list(map(opentag, newchildren))
            ).get_opcodes():
                if op != "equal":
                    for k in range(oi, oj):
                        print(
                            "{:5d} {:5s} - {}".format(
                                oldchildren[k].sourceline,
                                "",
                                oldcolor + tostring(oldchildren[k], max_length) + nocolor,
                            )
                        )
                    for k in range(ni, nj):
                        print(
                            "{:5s} {:5d} + {}".format(
                                "",
                                newchildren[k].sourceline,
                                newcolor + tostring(newchildren[k], max_length) + nocolor,
                            )
                        )
                else:
                    for ok, nk in zip(range(oi, oj), range(ni, nj)):
                        visit(oldchildren[ok], newchildren[nk])

    oldroot = oldtree.getroot()
    newroot = newtree.getroot()
    visit(oldroot, newroot)


if __name__ == "__main__":
    import argparse

    ap = argparse.ArgumentParser(description="Compare two XML files.")
    ap.add_argument("oldfile", help="first XML file")
    ap.add_argument("newfile", help="second XML file")
    ap.add_argument("-w", "--width", type=int, default=80, help="terminal width")
    args = ap.parse_args()

    max_length = args.width - 15

    try:
        oldtree = etree.parse(args.oldfile)
    except KeyboardInterrupt:
        pass
    except Exception as e:
        print("error: {}: {}".format(args.oldfile, e))
        exit(1)

    try:
        newtree = etree.parse(args.newfile)
    except KeyboardInterrupt:
        pass
    except Exception as e:
        print("error: {}: {}".format(args.newfile, e))
        exit(1)

    print("*****       - {}".format(args.oldfile))
    print("      ***** + {}".format(args.newfile))

    unified_diff(oldtree, newtree)


# bin/download_tacl.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020, Matt Post
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Reads through TACL papers in a specified XML file, downloads the PDF via the DOI
if not reachable on the Anthology, names appropriately.

https://www.mitpressjournals.org/doi/pdf/10.1162/tacl_a_00297
"""

import sys
import os

from anthology.utils import (
    deconstruct_anthology_id,
    infer_url,
    test_url_code,
    is_newstyle_id,
    retrieve_url,
)
from time import sleep

import lxml.etree as ET


def process_volume(anthology_volume):

    collection_id, volume_id, _ = deconstruct_anthology_id(anthology_volume)
    if is_newstyle_id(anthology_volume):
        venue_path = collection_id.split(".")[1]
    else:
        venue_path = os.path.join(collection_id[0], collection_id)

    print(f"Downloading PDFs for {anthology_volume}", file=sys.stderr)

    # Update XML
    xml_file = os.path.join(
        os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
    )
    tree = ET.parse(xml_file)

    for paper in tree.getroot().findall(f".//paper"):
        anthid = paper.find("./url").text

        # Try to get the URL from the Anthology
        if not test_url_code(infer_url(anthid)):
            doi = paper.find("./doi").text
            doi_pdf = f"https://www.mitpressjournals.org/doi/pdf/{doi}"
            local_path = os.path.join(
                args.anthology_files_dir, venue_path, f"{anthid}.pdf"
            )
            if not os.path.exists(os.path.dirname(local_path)):
                os.makedirs(os.path.dirname(local_path))

            retrieve_url(doi_pdf, local_path)
            print(f"Saved {doi_pdf} to {local_path}")
            sleep(1)


def main(args):
    for volume in args.anthology_volumes:
        process_volume(volume)


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "anthology_volumes",
        nargs="+",
        help="One or more Anthology volumes (e.g., P17-1, W18-02)",
    )
    parser.add_argument(
        "--anthology-files-dir",
        "-d",
        default=os.path.join(os.environ["HOME"], "anthology-files/pdf"),
        help="Anthology web directory root.",
    )
    args = parser.parse_args()

    main(args)


# bin/extract_corrections_for_processing.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Extracts corrections from the CSV output of our Attachments form at
https://forms.office.com/Pages/ResponsePage.aspx?id=DQSIkWdsW0yxEjajBLZtrQAAAAAAAAAAAAMAABqTSThUN0I2VEdZMTk4Sks3S042MVkxUEZQUVdOUS4u .
Each extracted line can then be passed directly to ./add_revision.py as arguments for ingestion.

e.g.,

    cat attachments.csv | ./extract_corrections_for_processing.py
    for file in corrections/*; do
      add_revision.py $(cat $file)
    done

Author: Matt Post
"""

import csv
import os
import sys

from anthology.utils import is_valid_id
from datetime import datetime


def main(args):

    for row in csv.DictReader(args.csv_file):
        #    ID,Start time,Completion time,Email,Name,Anthology ID,URL,Type,Explanation,Name,Email,License

        anthology_id = row["Anthology ID"].strip()
        download_path = row["URL"]
        attachment_type = row["Attachment type"]
        submitter_name = row["Your name"]
        submitter_email = row["Your email address"]
        submitted = row["Completion time"]
        explanation = row["Explanation"]

        date = datetime.strptime(submitted.split()[0], "%m/%d/%y").date().isoformat()

        if attachment_type not in ["Correction", "Erratum"]:
            print(f"Skipping {anthology_id} (type={attachment_type})", file=sys.stderr)
            continue

        if not is_valid_id(anthology_id):
            print(f"Bad anthology ID {anthology_id}", file=sys.stderr)

        outdir = os.path.join("corrections")
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        outfile = os.path.join(outdir, f"{anthology_id}.txt")
        if os.path.exists(outfile):
            print(
                f"* Refusing to overwrite existing file {outfile}, skipping",
                file=sys.stderr,
            )
            continue
        else:
            with open(outfile, "w") as out:
                erratum_flag = '' if attachment_type == 'Correction' else '-e '
                print(
                    f'{erratum_flag}{anthology_id} -d {date} "{download_path}" "{explanation}"',
                    file=out,
                )
                print(f"Wrote line for {anthology_id} to {outfile}", file=sys.stderr)


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("csv_file", type=argparse.FileType("r"))
    args = parser.parse_args()

    main(args)


# bin/find_name_variants.py

#!/usr/bin/env python3
# Marcel Bollmann <marcel@bollmann.me>, 2019

"""Usage: find_name_variants.py [--importdir=DIR]

Heuristically try to find variants of names not yet covered by name_variants.yaml
Finds names that slugify to the same thing.  Handles missing accents and
mistakes in first/last split, but not things like Tom/Thomas.  Prints output
that can be pasted into name_variants.yaml.

Options:
  --importdir=DIR          Directory to import XML files from. [default: {scriptdir}/../data/]
  -h, --help               Display this helpful text.
"""

from collections import defaultdict
from docopt import docopt
from slugify import slugify
import re
import os
import yaml, yamlfix

from anthology import Anthology
from anthology.people import PersonName


def score_variant(name):
    """Heuristically assign scores to names, with the idea of assigning higher
    scores to spellings more likely to be the correct canonical variant."""
    # Prefer longer variants
    score = len(name)
    # Prefer variants with non-ASCII characters
    score += sum((ord(c) > 127) for c in name)
    # Penalize upper-case characters after word boundaries
    score -= sum(any(c.isupper() for c in w[1:]) for w in re.split(r"\W+", name))
    # Penalize lower-case characters at word boundaries
    score -= sum(w[0].islower() if w else 0 for w in re.split(r"\W+", name))
    if name[0].islower():  # extra penalty for first name
        score -= 1

    return score


def to_dict(pn):
    return {"first": pn.first, "last": pn.last}


def main(anthology):
    variants = defaultdict(list)
    slugs = {}
    for person in anthology.people.personids():
        name = anthology.people.get_canonical_name(person)
        name_slug = slugify(repr(name))
        if name_slug in slugs:
            variants[slugs[name_slug]].append(repr(name))
        else:
            slugs[name_slug] = repr(name)

    canonical_variants = []
    for var1, varlist in variants.items():
        varlist.append(var1)
        # Determine best canonical variant
        scores = {var: score_variant(var) for var in varlist}
        canonical = max(scores.items(), key=lambda k: k[1])[0]
        canonical_variants.append(
            {
                "canonical": to_dict(PersonName.from_repr(canonical)),
                "variants": [
                    to_dict(PersonName.from_repr(var))
                    for var in varlist
                    if var != canonical
                ],
            }
        )

    canonical_variants.sort(
        key=lambda x: (x["canonical"]["last"], x["canonical"]["first"])
    )
    # flow style to match format of file name_variants.yaml
    print(yaml.dump(canonical_variants, allow_unicode=True, default_flow_style=None))


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )

    anthology = Anthology(importdir=args["--importdir"])
    main(anthology)


# bin/fix_attachments.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
One-time script used to remove duplicate attachments from the Anthology.
May be useful for salvaging in the future.

    python3 fix_attachments ../data/xml/*.xml
"""

import argparse
import os
import shutil
import ssl
import sys
import tempfile

from anthology.utils import indent

import lxml.etree as ET
import urllib.request


def main(args):

    for xml_file in args.files:
        # Update XML
        tree = ET.parse(xml_file)
        tree.getroot().tail = "\n"

        for paper in tree.getroot().findall(".//paper"):
            tail = paper.tail
            seen = []
            for attachment in paper.findall("./attachment"):
                if attachment.text in seen:
                    print(f"Removing: {attachment.text}")
                    paper.remove(attachment)
                seen.append(attachment.text)

            indent(paper, level=2)
            paper.tail = tail

        tree.write(xml_file, encoding="UTF-8", xml_declaration=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("files", nargs="+")
    args = parser.parse_args()

    main(args)


# bin/fix_titles.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Reads in a tab-delimited list of (Anth ID, new title) pairs, and sets
the titles for the papers. Efficiently opens each XML file only once
if inputs are ordered.

    cat new-titles.tsv | python3 ./fix_titles.py
"""

import argparse
import os
import sys

from anthology.utils import indent, deconstruct_anthology_id, make_simple_element
from normalize_anth import normalize

import lxml.etree as ET


def main(args):

    prev_xml_file = None
    tree = None
    for line in args.tsv_file:
        anth_id, newtitle = line.rstrip().split("\t")

        collection_id, volume_id, paper_id = deconstruct_anthology_id(anth_id)

        xml_file = os.path.join(
            os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
        )

        if prev_xml_file is not None and xml_file != prev_xml_file:
            print(f"-> Dumping to file {prev_xml_file}")
            indent(tree.getroot())
            tree.write(prev_xml_file, encoding="UTF-8", xml_declaration=True)

        if prev_xml_file is None or xml_file != prev_xml_file:
            tree = ET.parse(xml_file)

        volume = tree.getroot().find(f"./volume[@id='{volume_id}']")
        paper = volume.find(f"./paper[@id='{paper_id}']")
        if paper is None:
            print(f"Can't find {anth_id} in {xml_file}")
            continue

        title = paper.find(f"./title")
        if title is None:
            print(f"** WARNING: no title for for {anth_id}")
            continue

        print(f"{anth_id}: Changing title from\n-> {title.text} to\n-> {newtitle}")

        # Remove kids from existing title, update text, then re-normalize
        kids = [kid for kid in title]
        for kid in kids:
            title.remove(kid)
        title.text = newtitle
        normalize(title, informat="latex")

        prev_xml_file = xml_file

    indent(tree.getroot())
    tree.write(xml_file, encoding="UTF-8", xml_declaration=True)
    print(f"-> Dumping to file {xml_file}")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "tsv_file", nargs="?", default=sys.stdin, help="Where to read TSV file from"
    )
    args = parser.parse_args()

    main(args)


# bin/generate_crossref_doi_metadata.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Min-Yen Kan <kanmy@comp.nus.edu.sg>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Creates the Crossref metadata submission XML for the current (30
Jun 2019) Anthology XML data.  Needs to be uploaded to Crossref via
the ACL organization credentials.  Note that each DOI assigned costs
the ACL $1 USD.

- Only assign DOIs to ACL venues (i.e., those run by ACL or
  (co-)sponsored by an ACL chapter or SIG)
- Do not assign DOIs to journals that assign their own DOIs (as of
  current, both CL and TACL should be assigning their own DOIs)

See also https://github.com/acl-org/acl-anthology/wiki/DOI

Usage: python3 generate_crossref_doi_metadata.py <list of volume IDs>
e.g.,

    python3 generate_crossref_doi_metadata.py P19-1 P19-2 P19-3 P19-4 > acl2019_dois.xml

Limitations:
- This script does not inject the DOI data into the Anthology XML.
  For this, use `bin/add_dois.py <list of volume IDs>`.
- Doesn't properly handle existing ISBN information.
- Doesn't currently submit the frontmatter.

Tested:
- against 2018 workshop and conference data (working)
"""
import os
import re
import sys
import time

from lxml import etree

from anthology.utils import deconstruct_anthology_id, make_simple_element, is_newstyle_id
from anthology.data import CANONICAL_URL_TEMPLATE, DOI_PREFIX
from anthology.formatter import MarkupFormatter

# CONSTANTS
PUBLISHER_PLACE = "Stroudsburg, PA, USA"
DEPOSITOR_NAME = "Matt Post"
EMAIL_ADDRESS = "anthology@aclweb.org"
REGISTRANT = "Association for Computational Linguistics"
PUBLISHER = "Association for Computational Linguistics"
MONTH_HASH = {
    "January": "1",
    "February": "2",
    "March": "3",
    "April": "4",
    "May": "5",
    "June": "6",
    "July": "7",
    "August": "8",
    "Aug": "8",
    "September": "9",
    "October": "10",
    "November": "11",
    "December": "12",
}

# FUNCTION DEFINITIONS
def prettify(elem):
    """Return a pretty-printed XML string for the Element."""
    rough_string = etree.tostring(elem, "utf-8")
    reparsed = minidom.parseString(rough_string)
    return reparsed.toprettyxml(indent="  ")


def main(volumes):

    formatter = MarkupFormatter()

    ## Assemble container
    doi_batch = make_simple_element(
        "doi_batch",
        attrib={
            "xmlns": "http://www.crossref.org/schema/4.4.1",
            "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation": "http://www.crossref.org/schema/4.4.1 http://www.crossref.org/schema/deposit/crossref4.4.1.xsd",
            "version": "4.4.1",
        },
        namespaces={"xsi": "http://www.w3.org/2001/XMLSchema-instance"},
    )
    new_volume = etree.ElementTree(doi_batch)

    ## Assemble head
    head = make_simple_element("head", parent=new_volume.getroot())
    dbi = make_simple_element("doi_batch_id", text=str(int(time.time())), parent=head)

    timestamp = make_simple_element("timestamp", text=str(int(time.time())), parent=head)

    depositor = make_simple_element("depositor", parent=head)
    depositor_name = make_simple_element(
        "depositor_name", text=DEPOSITOR_NAME, parent=depositor
    )
    email_address = make_simple_element(
        "email_address", text=EMAIL_ADDRESS, parent=depositor
    )

    registrant = make_simple_element("registrant", text=REGISTRANT, parent=head)

    ## Assemble body
    body = make_simple_element("body", parent=new_volume.getroot())

    year = ""
    start_month = ""
    end_month = ""

    for full_volume_id in sorted(volumes):
        collection_id, volume_id, _ = deconstruct_anthology_id(full_volume_id)

        collection_file = os.path.join(
            os.path.dirname(sys.argv[0]), "..", "data", "xml", f"{collection_id}.xml"
        )
        tree = etree.parse(collection_file)

        v = tree.getroot().find(f"./volume[@id='{volume_id}']")
        if v is None:
            print(f"* Can't find volume {full_volume_id}", file=sys.stderr)
            continue

        ## Assemble frontmatter
        c = make_simple_element("conference", parent=body)
        contribs = make_simple_element("contributors", parent=c)
        editor_index = 0

        meta = v.find("./meta")
        for tag in meta:
            if tag.tag == "year":
                year = tag.text
            elif tag.tag == "month":
                month = tag.text
                try:
                    start_month = MONTH_HASH[re.split("[-–]", month)[0]]
                    end_month = MONTH_HASH[re.split("[-–]", month)[1]]
                except IndexError as e:  # only one month
                    start_month = MONTH_HASH[month]
                    end_month = MONTH_HASH[month]
                except Exception as e:
                    print(
                        f"FATAL: can't parse month {month} in {full_volume_id}",
                        file=sys.stderr,
                    )
                    sys.exit(1)
            elif tag.tag == "booktitle":
                booktitle = formatter.as_text(tag)
            elif tag.tag == "address":
                address = tag.text
            elif tag.tag == "publisher":
                publisher = tag.text
            elif tag.tag == "editor":
                pn = make_simple_element(
                    "person_name",
                    parent=contribs,
                    attrib={
                        "contributor_role": "chair",
                        "sequence": "first" if editor_index == 0 else "additional",
                    },
                )
                editor_index += 1

                for name_part in tag:
                    # Check if empty (e.g., "Mausam")
                    if (
                        name_part.tag == "first"
                        and name_part.text is not None
                        and name_part.text != ""
                    ):
                        gn = make_simple_element(
                            "given_name", parent=pn, text=name_part.text
                        )
                    elif name_part.tag == "last":
                        sn = make_simple_element(
                            "surname", text=name_part.text, parent=pn
                        )

        if editor_index == 0:
            print(f"FATAL: Found no editors", file=sys.stderr)
            sys.exit(1)

        # Assemble Event Metadata
        em = make_simple_element("event_metadata", parent=c)
        cn = make_simple_element("conference_name", parent=em, text=booktitle)
        cl = make_simple_element("conference_location", parent=em, text=address)
        cd = make_simple_element(
            "conference_date",
            parent=em,
            attrib={
                "start_year": year,
                "end_year": year,
                "start_month": start_month,
                "end_month": end_month,
            },
        )

        # Assemble Proceedings Metadata
        pm = make_simple_element(
            "proceedings_metadata", parent=c, attrib={"language": "en"}
        )
        pt = make_simple_element("proceedings_title", parent=pm, text=booktitle)
        p = make_simple_element("publisher", parent=pm)
        pn = make_simple_element("publisher_name", parent=p, text=publisher)
        pp = make_simple_element("publisher_place", parent=p, text=PUBLISHER_PLACE)
        pd = make_simple_element("publication_date", parent=pm)
        y = make_simple_element("year", parent=pd, text=year)
        noisbn = make_simple_element(
            "noisbn", parent=pm, attrib={"reason": "simple_series"}
        )

        # DOI assignation data
        dd = make_simple_element("doi_data", parent=pm)
        doi = make_simple_element("doi", parent=dd, text=DOI_PREFIX + full_volume_id)
        resource = make_simple_element(
            "resource", parent=dd, text=CANONICAL_URL_TEMPLATE.format(full_volume_id)
        )

        for paper in v.findall("./paper"):
            ## Individual Paper Data
            paper_id = paper.attrib["id"]
            if paper.find("./url") is not None:
                url = paper.find("./url").text
            else:
                if is_newstyle_id(full_volume_id):
                    url = f"{full_volume_id}.{paper_id}"
                elif len(full_volume_id) == 6:
                    url = f"{full_volume_id}{paper_id:02d}"
                elif len(full_volume_id) == 5:
                    url = f"{full_volume_id}{paper_id:03d}"

            cp = make_simple_element("conference_paper", parent=c)

            # contributors
            contribs = make_simple_element("contributors", parent=cp)
            author_index = 0
            for author in paper.findall("./author"):
                pn = make_simple_element(
                    "person_name",
                    parent=contribs,
                    attrib={
                        "contributor_role": "author",
                        "sequence": "first" if author_index == 0 else "additional",
                    },
                )
                author_index += 1

                for name_part in author:
                    if (
                        name_part.tag == "first"
                        and name_part.text is not None
                        and name_part.text != ""
                    ):
                        gn = make_simple_element(
                            "given_name", parent=pn, text=name_part.text
                        )
                    elif name_part.tag == "last":
                        sn = make_simple_element(
                            "surname", text=name_part.text, parent=pn
                        )

            for title in paper.iter(tag="title"):
                o_titles = make_simple_element("titles", parent=cp)
                o_title = make_simple_element(
                    "title", parent=o_titles, text=formatter.as_text(title)
                )

            pd = make_simple_element("publication_date", parent=cp)
            o_year = make_simple_element("year", parent=pd)
            o_year.text = year

            for pages in paper.iter(tag="pages"):
                o_pages = make_simple_element("pages", parent=cp)
                fp = make_simple_element("first_page", parent=o_pages)
                lp = make_simple_element("last_page", parent=o_pages)
                try:
                    fp.text = re.split("[-–]", pages.text)[0]
                    lp.text = re.split("[-–]", pages.text)[1]
                except IndexError as e:  # only one page
                    fp.text = pages.text
                    lp.text = pages.text

            # DOI assignation data
            dd = make_simple_element("doi_data", parent=cp)
            doi = make_simple_element("doi", parent=dd, text=DOI_PREFIX + url)
            resource = make_simple_element(
                "resource", parent=dd, text=CANONICAL_URL_TEMPLATE.format(url)
            )

    print(
        etree.tostring(
            new_volume,
            pretty_print=True,
            encoding="UTF-8",
            xml_declaration=True,
            with_tail=True,
        ).decode("utf-8")
    )


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("volumes", nargs="+", help="Volumes to add DOIs to")
    args = parser.parse_args()

    main(args.volumes)


# bin/indent.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Author: Matt Post <post@cs.jhu.edu>

Produces two-space indenting for all items except
<author>, <editor>, <title>, and <booktitle> which
remain on one line.

Usage: indent.py <in.xml> <out.xml>
"""

import lxml.etree as etree
import argparse
import re
import sys

from anthology.utils import indent

if __name__ == "__main__":

    infilename = sys.argv[1]
    outfilename = sys.argv[2]

    tree = etree.parse(infilename)
    root = tree.getroot()
    indent(root)

    tree.write(outfilename, encoding="UTF-8", xml_declaration=True, with_tail=True)


# bin/ingest.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019, 2020 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Ingests data into the Anthology. It takes a list of one or more
ACLPUB proceedings/ directories and does the following:

- executes some basic sanity checks
- applies normalization to names and titles (e.g, fixed-case protection)
- generates the nexted XML in the Anthology repository
- copies the PDFs and attachments into place for rsyncing to the server

Updated in March 2020, this script replaces:
- the old ingest.py (which converted the old ACLPUB flat XML format)
- anthologize.pl in ACLPUB
- anthology_xml.py in ACLPUB

"""

import argparse
import iso639
import os
import re
import readline
import shutil
import sys

import lxml.etree as etree

from collections import defaultdict, OrderedDict
from datetime import datetime
from glob import glob

from normalize_anth import normalize
from anthology.bibtex import read_bibtex
from anthology.index import AnthologyIndex
from anthology.people import PersonName
from anthology.sigs import SIGIndex
from anthology.utils import (
    make_simple_element,
    build_anthology_id,
    deconstruct_anthology_id,
    indent,
    compute_hash_from_file,
)
from anthology.venues import VenueIndex

from itertools import chain
from typing import Dict, Any

from slugify import slugify


def log(text: str, fake: bool = False):
    message = "[DRY RUN] " if fake else ""
    print(f"{message}{text}", file=sys.stderr)


def load_bibkeys(anthology_datadir):
    bibkeys = set()
    for xmlfile in glob(os.path.join(anthology_datadir, "xml", "*.xml")):
        tree = etree.parse(xmlfile)
        root = tree.getroot()
        bibkeys.update(str(elem.text) for elem in root.iterfind(".//bibkey"))
    return bibkeys


def read_meta(path: str) -> Dict[str, Any]:
    meta = {"chairs": []}
    with open(path) as instream:
        for line in instream:
            if re.match(r"^\s*$", line):
                continue
            key, value = line.rstrip().split(" ", maxsplit=1)
            if key.startswith("chair"):
                meta["chairs"].append(value)
            else:
                meta[key] = value
    if "volume" in meta and re.match(rf"^[a-z0-9]+$", meta["volume"]) is None:
        raise Exception(f"Invalid volume key '{meta['volume']}' in {path}")

    return meta


def maybe_copy(source_path, dest_path):
    """Copies the file if it's different from the target."""
    if not os.path.exists(dest_path) or compute_hash_from_file(
        source_path
    ) != compute_hash_from_file(dest_path):
        log(f"Copying {source_path} -> {dest_path}", args.dry_run)
        shutil.copyfile(source_path, dest_path)


def bib2xml(bibfilename, anthology_id):
    """
    Moved here from ACLPUB's anthology_xml.py script.
    """

    fields = [
        'title',
        'author',
        'editor',
        'booktitle',
        'month',
        'year',
        'address',
        'publisher',
        'pages',
        'abstract',
        'url',
        'doi',
        'language',
    ]

    try:
        collection_id, volume_name, paper_no = deconstruct_anthology_id(anthology_id)
    except ValueError:
        print(f"Couldn't split {anthology_id}", file=sys.stderr)
        sys.exit(1)
    if paper_no == '':
        return  # skip the master bib file; we only process the individual files

    bibdata = read_bibtex(bibfilename)
    if len(bibdata.entries) != 1:
        log(f"more than one entry in {bibfilename}")
    bibkey, bibentry = bibdata.entries.items()[0]
    if len(bibentry.fields) == 0:
        log(f"parsing bib of paper {paper_no} failed")
        sys.exit(1)

    paper = make_simple_element("paper", attrib={"id": paper_no})
    for field in list(bibentry.fields) + list(bibentry.persons):
        if field not in fields:
            log(f"unknown field {field}")

    for field in fields:
        if field in ['author', 'editor']:
            if field in bibentry.persons:
                for person in bibentry.persons[field]:
                    first_text = ' '.join(person.bibtex_first_names)
                    last_text = ' '.join(person.prelast_names + person.last_names)
                    if person.lineage_names:
                        last_text += ', ' + ' '.join(person.lineage_names)

                    # Don't distinguish between authors that have only a first name
                    # vs. authors that have only a last name; always make it a last name.
                    if last_text.strip() in [
                        '',
                        '-',
                    ]:  # Some START users have '-' for null
                        last_text = first_text
                        first_text = ''

                    name_node = make_simple_element(field, parent=paper)
                    make_simple_element("first", first_text, parent=name_node)
                    make_simple_element("last", last_text, parent=name_node)
        else:
            if field == 'url':
                value = f"{anthology_id}"
            elif field in bibentry.fields:
                value = bibentry.fields[field]
            elif field == 'bibtype':
                value = bibentry.type
            elif field == 'bibkey':
                value = bibkey
            else:
                continue

            try:
                make_simple_element(field, text=value, parent=paper)
            except:
                print(
                    f"Couldn't process {bibfilename} for {anthology_id}", file=sys.stderr
                )
                sys.exit(2)

    return paper


def main(args):
    volumes = {}

    anthology_datadir = os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
    venue_index = VenueIndex(srcdir=anthology_datadir)
    venue_keys = [venue["slug"].lower() for _, venue in venue_index.items()]

    sig_index = SIGIndex(srcdir=anthology_datadir)

    people = AnthologyIndex(srcdir=anthology_datadir)
    people.bibkeys = load_bibkeys(anthology_datadir)

    def correct_caps(name):
        """
        Many people submit their names in "ALL CAPS" or "all lowercase".
        Correct this with heuristics.
        """
        if name.islower() or name.isupper():
            # capitalize all parts
            corrected = " ".join(list(map(lambda x: x.capitalize(), name.split())))
            print(
                f"-> Correcting capitalization of '{name}' to '{corrected}'",
                file=sys.stderr,
            )
            name = corrected

        return name

    def disambiguate_name(node, anth_id):
        name = PersonName.from_element(node)
        ids = people.get_ids(name)
        choice = -1
        if len(ids) > 1:
            while choice < 0 or choice >= len(ids):
                print(
                    f"({anth_id}): ambiguous author {name}; Please choose from the following:"
                )
                for i, id_ in enumerate(ids):
                    print(f"[{i}] {id_} ({people.get_comment(id_)})")
                choice = int(input("--> "))

        return ids[choice], choice

    # Build list of volumes, confirm uniqueness
    unseen_venues = []
    for proceedings in args.proceedings:
        meta = read_meta(os.path.join(proceedings, "meta"))

        venue_abbrev = meta["abbrev"]
        venue_slug = venue_index.get_slug(venue_abbrev)

        if str(datetime.now().year) in venue_abbrev:
            print(f"Fatal: Venue assembler put year in acronym: '{venue_abbrev}'")
            sys.exit(1)

        if re.match(r".*\d$", venue_abbrev) is not None:
            print(
                f"WARNING: Venue {venue_abbrev} ends in a number, this is probably a mistake"
            )

        if venue_slug not in venue_keys:
            unseen_venues.append((venue_slug, venue_abbrev, meta["title"]))

        meta["path"] = proceedings

        meta["collection_id"] = collection_id = meta["year"] + "." + venue_slug
        volume_name = meta["volume"].lower()
        volume_full_id = f"{collection_id}-{volume_name}"

        if volume_full_id in volumes:
            print("Error: ")

        volumes[volume_full_id] = meta

        if "sig" in meta:
            print(
                f"Add this line to {anthology_datadir}/sigs/{meta['sig'].lower()}.yaml:"
            )
            print(f"  - {meta['year']}:")
            print(f"    - {volume_full_id} # {meta['booktitle']}")

    # Make sure all venues exist
    if len(unseen_venues) > 0:
        for venue in unseen_venues:
            slug, abbrev, title = venue
            print(f"Creating venue '{abbrev}' ({title})")
            venue_index.add_venue(abbrev, title)
        venue_index.dump(directory=anthology_datadir)

    # Copy over the PDFs and attachments
    for volume_full_id, meta in volumes.items():
        root_path = os.path.join(meta["path"], "cdrom")
        collection_id = meta["collection_id"]
        venue_name = meta["abbrev"].lower()
        volume_name = meta["volume"].lower()
        year = meta["year"]

        pdfs_dest_dir = os.path.join(args.pdfs_dir, venue_name)
        if not os.path.exists(pdfs_dest_dir):
            os.makedirs(pdfs_dest_dir)

        def find_book():
            """Book location has shifted a bit over the years"""

            potential_names = [
                os.path.join(meta["path"], "book.pdf"),
                os.path.join(
                    meta["path"],
                    "cdrom",
                    f"{year}-{venue_name.lower()}-{volume_name}.pdf",
                ),
                os.path.join(meta["path"], "cdrom", f"{venue_name.upper()}-{year}.pdf"),
            ]

            for book_rel_path in potential_names:
                if os.path.exists(book_rel_path):
                    return book_rel_path

            return None

        # copy the book from the top-level proceedings/ dir, named "VENUE-year.pdf",
        # or sometimes "book.pdf"
        book_src_path = find_book()
        book_dest_path = None
        if book_src_path is not None and not args.dry_run:
            book_dest_path = (
                os.path.join(pdfs_dest_dir, f"{collection_id}-{volume_name}") + ".pdf"
            )
            maybe_copy(book_src_path, book_dest_path)

        # temp holder for papers in each volume
        volume = dict()

        # copy the paper PDFs
        pdf_src_dir = os.path.join(root_path, "pdf")
        for pdf_file in os.listdir(pdf_src_dir):
            # Skip . files
            if os.path.basename(pdf_file).startswith("."):
                continue

            # names are {abbrev}{number}.pdf
            match = re.match(rf".*\.(\d+)\.pdf", pdf_file)

            if match is not None:
                paper_num = int(match[1])
                paper_id_full = f"{collection_id}-{volume_name}.{paper_num}"

                bib_path = os.path.join(
                    root_path,
                    "bib",
                    pdf_file.replace("/pdf", "/bib/").replace(".pdf", ".bib"),
                )

                pdf_src_path = os.path.join(pdf_src_dir, pdf_file)
                pdf_dest_path = os.path.join(
                    pdfs_dest_dir, f"{collection_id}-{volume_name}.{paper_num}.pdf"
                )
                if not args.dry_run:
                    maybe_copy(pdf_src_path, pdf_dest_path)

                volume[paper_num] = {
                    "anthology_id": paper_id_full,
                    "bib": bib_path,
                    "pdf": pdf_dest_path,
                    "attachments": [],
                }

        # copy the attachments
        if os.path.exists(os.path.join(root_path, "additional")):
            attachments_dest_dir = os.path.join(args.attachments_dir, venue_name)
            if not os.path.exists(attachments_dest_dir):
                os.makedirs(attachments_dest_dir)
            for attachment_file in os.listdir(os.path.join(root_path, "additional")):
                if os.path.basename(attachment_file).startswith("."):
                    continue
                attachment_file_path = os.path.join(
                    root_path, "additional", attachment_file
                )
                # Find the attachment file, using a bit of a fuzzy
                # match. The fuzzy match is because sometimes people
                # generate the proceedings with the wrong venue
                # code. If we correct it, we still need to be able to
                # find the file.
                match = re.match(
                    rf"{year}\..*-\w+\.(\d+)_?(\w+)\.(\w+)$", attachment_file
                )
                if match is None:
                    print(
                        f"* Warning: no attachment match for {attachment_file}",
                        file=sys.stderr,
                    )
                    sys.exit(2)

                paper_num, type_, ext = match.groups()
                paper_num = int(paper_num)

                file_name = f"{collection_id}-{volume_name}.{paper_num}.{type_}.{ext}"
                dest_path = os.path.join(attachments_dest_dir, file_name)
                if not args.dry_run and not os.path.exists(dest_path):
                    log(f"Copying {attachment_file} -> {dest_path}", args.dry_run)
                    shutil.copyfile(attachment_file_path, dest_path)

                volume[paper_num]["attachments"].append((dest_path, type_))

        # create xml
        collection_file = os.path.join(
            args.anthology_dir, "data", "xml", f"{collection_id}.xml"
        )
        if os.path.exists(collection_file):
            root_node = etree.parse(collection_file).getroot()
        else:
            root_node = make_simple_element("collection", attrib={"id": collection_id})

        volume_node = make_simple_element(
            "volume",
            attrib={"id": volume_name, "ingest-date": args.ingest_date},
        )

        # Replace the existing one if present
        existing_volume_node = root_node.find(f"./volume[@id='{volume_name}']")
        for i, child in enumerate(root_node):
            if child.attrib["id"] == volume_name:
                root_node[i] = volume_node
                break
        else:
            root_node.append(volume_node)

        meta_node = None

        for paper_num, paper in sorted(volume.items()):
            paper_id_full = paper["anthology_id"]
            bibfile = paper["bib"]
            paper_node = bib2xml(bibfile, paper_id_full)

            if paper_node.attrib["id"] == "0":
                # create metadata subtree
                meta_node = make_simple_element("meta", parent=volume_node)
                title_node = paper_node.find("title")
                title_node.tag = "booktitle"
                meta_node.append(title_node)
                for author_or_editor in chain(
                    paper_node.findall("./author"), paper_node.findall("./editor")
                ):
                    disamb_name, name_choice = disambiguate_name(
                        author_or_editor, paper_id_full
                    )
                    if name_choice != -1:
                        author_or_editor.attrib["id"] = disamb_name
                    person = PersonName.from_element(author_or_editor)
                    for name_part in author_or_editor:
                        name_part.text = correct_caps(name_part.text)
                    meta_node.append(author_or_editor)
                    author_or_editor.tag = "editor"

                # Here, we grab the publisher from the meta file, in case it's not in the
                # frontmatter paper. We don't handle the situation where it's in neither!
                publisher_node = paper_node.find("publisher")
                if publisher_node is None:
                    publisher_node = make_simple_element("publisher", meta["publisher"])
                meta_node.append(publisher_node)

                # Look for the address in the bib file, then the meta file
                address_node = paper_node.find("address")
                if address_node is None:
                    address_node = make_simple_element("address", meta["location"])
                meta_node.append(address_node)

                meta_node.append(paper_node.find("month"))
                meta_node.append(paper_node.find("year"))
                if book_dest_path is not None:
                    make_simple_element(
                        "url",
                        text=f"{collection_id}-{volume_name}",
                        attrib={"hash": compute_hash_from_file(book_dest_path)},
                        parent=meta_node,
                    )

                # modify frontmatter tag
                paper_node.tag = "frontmatter"
                del paper_node.attrib["id"]
            else:
                # remove unneeded fields
                for child in paper_node:
                    if child.tag in [
                        "editor",
                        "address",
                        "booktitle",
                        "publisher",
                        "year",
                        "month",
                    ]:
                        paper_node.remove(child)

            url = paper_node.find("./url")
            if url is not None:
                url.attrib["hash"] = compute_hash_from_file(paper["pdf"])

            for path, type_ in paper["attachments"]:
                make_simple_element(
                    "attachment",
                    text=os.path.basename(path),
                    attrib={
                        "type": type_,
                        "hash": compute_hash_from_file(path),
                    },
                    parent=paper_node,
                )

            if len(paper_node) > 0:
                volume_node.append(paper_node)

            # Normalize
            for oldnode in paper_node:
                normalize(oldnode, informat="latex")

            # Adjust the language tag
            language_node = paper_node.find("./language")
            if language_node is not None:
                try:
                    lang = iso639.languages.get(name=language_node.text)
                except KeyError:
                    raise Exception(f"Can't find language '{language_node.text}'")
                language_node.text = lang.part3

            # Fix author names
            for name_node in chain(
                paper_node.findall("./author"), paper_node.findall("./editor")
            ):
                disamb_name, name_choice = disambiguate_name(name_node, paper_id_full)
                if name_choice != -1:
                    name_node.attrib["id"] = disamb_name
                person = PersonName.from_element(name_node)
                for name_part in name_node:
                    name_part.text = correct_caps(name_part.text)

        # Other data from the meta file
        if "isbn" in meta:
            make_simple_element("isbn", meta["isbn"], parent=meta_node)

        indent(root_node)
        tree = etree.ElementTree(root_node)
        tree.write(
            collection_file, encoding="UTF-8", xml_declaration=True, with_tail=True
        )


if __name__ == "__main__":
    now = datetime.now()
    today = f"{now.year}-{now.month:02d}-{now.day:02d}"

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "proceedings", nargs="+", help="List of paths to ACLPUB proceedings/ directories."
    )
    parser.add_argument(
        "--ingest-date",
        "-d",
        type=str,
        default=today,
        help="Ingestion date as YYYY-MM-DD. Default: %(default)s.",
    )
    anthology_path = os.path.join(os.path.dirname(sys.argv[0]), "..")
    parser.add_argument(
        "--anthology-dir",
        "-r",
        default=anthology_path,
        help="Root path of ACL Anthology Github repo. Default: %(default)s.",
    )
    pdfs_path = os.path.join(os.environ["HOME"], "anthology-files", "pdf")
    parser.add_argument(
        "--pdfs-dir", "-p", default=pdfs_path, help="Root path for placement of PDF files"
    )
    attachments_path = os.path.join(os.environ["HOME"], "anthology-files", "attachments")
    parser.add_argument(
        "--attachments-dir",
        "-a",
        default=attachments_path,
        help="Root path for placement of PDF files",
    )
    parser.add_argument(
        "--dry-run", "-n", action="store_true", help="Don't actually copy anything."
    )
    args = parser.parse_args()

    main(args)


# bin/ingest_archive_video.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2021 Xinru Yan <xinru1414@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Usage:
#   python -v PATH_TO_VIDEO_DIR
#
# Video dir contains a list of videos need to be ingested, for example
#
# videos/
#    N13-1001.mp4
#    N13-1002.mp4
#    ...
#
#

import click
import os, glob
import lxml.etree as et
import argparse
from typing import List, Tuple
from anthology.utils import deconstruct_anthology_id, make_simple_element, indent

ROOT = os.path.dirname(os.path.abspath(__file__))
DATA_DIR = os.path.join(ROOT, "../data/xml")


def get_collection_ids(video_dir: str) -> List[str]:
    '''
    Go over all the .mp4 files in the video dir and extract unique collection ids, which will be used to identify xmls that need to be updated.

    param:
    video_dir: directory contains video files, eg: /Users/xinruyan/Dropbox/naacl-2013/

    return:
    a list of unique collection ids, eg: ['N13', 'Q13']
    '''
    collection_ids = list(
        set(
            [
                deconstruct_anthology_id(file[len(video_dir) :].split('.')[0])[0]
                for file in glob.glob(f"{video_dir}/*.mp4")
            ]
        )
    )
    return collection_ids


def get_anth_ids(video_dir: str) -> Tuple[List[str], List[List[str]]]:
    '''
    Go over all the .mp4 files in the video dir and extract two types of anthology ids, which will be used to identify papers that needs video tag.

    param:
    video_dir: directory contains video files, eg: /Users/xinruyan/Dropbox/naacl-2013/

    return:
    a tuple, (anth_ids_single, anth_ids_multiple),
    anth_ids_single: a list of anth_ids which only has one video to ingest, eg: ['N13-1118', 'N13-1124']
    anth_ids_multiple: a list of list of [anth_ids], [vid_num] which has multiple videos to ingest, eg: [['N13-4001', '1'],['N13-4001', '2'],['N13-4002', '1'],['N13-4002', '2']. vid_num represents the numbered videos.
    '''
    anth_ids_single = [
        file[len(video_dir) :].split('.')[0]
        for file in glob.glob(f"{video_dir}/*.mp4")
        if len(file[len(video_dir) :].split('.')) == 2
    ]
    anth_ids_multiple = [
        file[len(video_dir) :].split('.')[0:-1]
        for file in glob.glob(f"{video_dir}/*.mp4")
        if len(file[len(video_dir) :].split('.')) > 2
    ]
    anth_ids_multiple.sort()
    return anth_ids_single, anth_ids_multiple


def add_video_tag_single(anth_id, xml_parse):
    '''
    Add video tag for paper f'{anth_id}'
    '''
    collection_id, volume_id, paper_id = deconstruct_anthology_id(anth_id)
    paper = xml_parse.find(f'./volume[@id="{volume_id}"]/paper[@id="{paper_id}"]')
    video_url = anth_id + '.mp4'
    make_simple_element("video", attrib={"href": video_url}, parent=paper)


def add_video_tag_multiple(anth_id, vid_num, xml_parse):
    '''
    Add video tag for paper f`{anth_id}` with multiple number of videos
    Adapted from add_video_tags.py
    '''
    collection_id, volume_id, paper_id = deconstruct_anthology_id(anth_id)
    paper = xml_parse.find(f'./volume[@id="{volume_id}"]/paper[@id="{paper_id}"]')
    video_url = anth_id + f'.{vid_num}' + '.mp4'
    make_simple_element("video", attrib={"href": video_url}, parent=paper)


def update_xml(data_dir, collection_id, extention, xml_tree):
    '''
    Update xml
    Adapted from add_video_tags.py
    '''
    with open(os.path.join(data_dir, collection_id + extention), 'wb') as f:
        indent(xml_tree.getroot())
        xml_tree.write(f, encoding="UTF-8", xml_declaration=True)


@click.command()
@click.option(
    '-v',
    '--video_dir',
    default='/Users/xinruyan/Dropbox/naacl-2013/',
    help='Directory contains all videos need to be ingested',
)
def main(video_dir):
    collection_ids = get_collection_ids(video_dir=video_dir)

    xml_files = [
        file
        for file in os.listdir(DATA_DIR)
        if os.path.splitext(file)[0] in collection_ids
    ]

    anth_ids_single, anth_ids_multiple = get_anth_ids(video_dir=video_dir)

    for file in xml_files:
        collection_id, extention = os.path.splitext(file)
        tree = et.parse(os.path.join(DATA_DIR, file))
        for anth_id in anth_ids_single:
            if collection_id in anth_id:
                add_video_tag_single(anth_id=anth_id, xml_parse=tree)
                update_xml(
                    data_dir=DATA_DIR,
                    collection_id=collection_id,
                    extention=extention,
                    xml_tree=tree,
                )

        for anth_id_vid_num in anth_ids_multiple:
            anth_id = anth_id_vid_num[0]
            vid_num = anth_id_vid_num[1]
            if collection_id in anth_id:
                add_video_tag_multiple(anth_id=anth_id, vid_num=vid_num, xml_parse=tree)
                update_xml(
                    data_dir=DATA_DIR,
                    collection_id=collection_id,
                    extention=extention,
                    xml_tree=tree,
                )


if __name__ == '__main__':
    main()


# bin/ingest_lilt.py

#!/usr/bin/env python3
"""
Quick script for ingesting LiLT metadata.
Author: Matt Post
July 2020
"""

import csv
import lxml.etree as etree
import os
import shutil
import ssl
import sys
import urllib.request

import anthology
from anthology.utils import make_simple_element, indent, compute_hash_from_file
from datetime import datetime
from normalize_anth import normalize
from likely_name_split import NameSplitter


def dump_collection(tree, collection_file):
    indent(tree.getroot())
    tree.write(collection_file, encoding="UTF-8", xml_declaration=True, with_tail=True)


def main(args):
    anth = anthology.Anthology(importdir=os.path.join(args.anthology, "data"))
    splitter = NameSplitter(anth)

    paper_nums = {}
    venue = "lilt"
    prev_year = None
    prev_volume = None
    for row in csv.DictReader(args.tsv_file, delimiter='\t'):
        year = row.get("year")
        month = row.get("month")
        issue = row.get("issue#", "")
        abstract = row.get("abstract")
        collection_id = f"{year}.lilt"
        if year != prev_year:
            if prev_year is not None:
                dump_collection(
                    tree,
                    os.path.join(args.anthology, "data", "xml", f"{prev_year}.lilt.xml"),
                )

            tree = etree.ElementTree(
                make_simple_element("collection", attrib={"id": collection_id})
            )
            root = tree.getroot()
        prev_year = year

        volume_name = row.get("Volume#")
        if volume_name != prev_volume:
            volume = make_simple_element(
                "volume", attrib={"id": volume_name}, parent=root
            )
            meta = make_simple_element("meta", parent=volume)
            make_simple_element("booktitle", row.get("Booktitle"), parent=meta)
            make_simple_element("publisher", "CSLI Publications", parent=meta)
            make_simple_element("year", year, parent=meta)
            if month:
                make_simple_element("month", month, parent=meta)

        paper_num = paper_nums[volume_name] = paper_nums.get(volume_name, 0) + 1

        prev_volume = volume_name

        paper = make_simple_element("paper", attrib={"id": str(paper_num)}, parent=volume)
        paper_id = f"{collection_id}-{volume_name}.{paper_num}"
        make_simple_element("title", row.get("title"), parent=paper)
        authors = row.get("authors")
        for author_name in authors.split(" and "):
            author = make_simple_element("author", parent=paper)
            surname, givenname = splitter.best_split(author_name)
            make_simple_element("first", givenname, parent=author)
            make_simple_element("last", surname, parent=author)

        if abstract != "":
            make_simple_element("abstract", abstract, parent=paper)
        if issue != "":
            make_simple_element("issue", issue, parent=paper)

        for node in paper:
            normalize(node, "latex")

        dest_dir = f"{args.anthology_files_path}/lilt"
        if not os.path.exists(dest_dir):
            os.makedirs(dest_dir)

        source_path = os.path.join(
            "pdf", row.get("PDF").replace("\\", "/").replace("../", "")
        )
        if os.path.exists(source_path):
            dest_path = os.path.join(
                dest_dir, f"{collection_id}-{volume_name}.{paper_num}.pdf"
            )
            shutil.copy(source_path, dest_path)
            print(f"Copying {source_path} to {dest_path}", file=sys.stderr)
            os.chmod(dest_path, 0o644)
            checksum = compute_hash_from_file(dest_path)
            make_simple_element("url", paper_id, attrib={"hash": checksum}, parent=paper)

    dump_collection(
        tree, os.path.join(args.anthology, "data", "xml", f"{collection_id}.xml")
    )


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('tsv_file', type=argparse.FileType("r"))
    parser.add_argument(
        '--anthology',
        default=f"{os.environ.get('HOME')}/code/acl-anthology",
        help="Path to Anthology repo (cloned from https://github.com/acl-org/acl-anthology)",
    )
    parser.add_argument(
        '--anthology-files-path',
        default=f"{os.environ.get('HOME')}/anthology-files/pdf",
        help="Path to Anthology files (Default: ~/anthology-files",
    )
    args = parser.parse_args()

    main(args)


# bin/ingest_mitpress.py

#! /usr/bin/env python3
"""
Convert MIT Press XML files for CL and TACL to Anthology XML.

version 0.5 - reads from new MIT Press format.
version 0.4 - now updates XML directly, skips existing papers, sorts by page number
version 0.3 - produces anthology ID in new format 2020.cl-1.1

Example usage: unpack the ZIP file from MIT press. You'll have something like this:

    ./taclv9-11082021/
      ./xml/
        tacl_a_00350.xml
        tacl_a_00351.xml
        tacl_a_00352.xml
        ...
      ./assets/
        tacl_a_00350.pdf
        tacl_a_00351.pdf
        tacl_a_00352.pdf
        ...

Then, run

    /path/to/anthology/bin/ingest_mitpress.py /path/to/taclv9-11082021/

This will

* infer the path of, then update or create the XML file, skipping existing papers
* copy new PDFs where they can be bundled up or rsynced over.

It assumes that you are working within a single collection (e.g., a single XML
file), but there can be multiple volumes (like for CL).

Warning (August 2020): not yet tested with CL, but should work!

Authors: Arya D. McCarthy, Matt Post
"""
import os
import shutil
import logging
import lxml.etree as etree

from pathlib import Path
from typing import List, Optional, Tuple

from anthology import Anthology, Paper, Volume
from normalize_anth import normalize
from anthology.utils import make_simple_element, indent, compute_hash_from_file

__version__ = "0.5"

TACL = "tacl"
CL = "cl"


def collapse_spaces(text: str) -> str:
    return " ".join(text.split())


def get_volume_info(xml: Path) -> str:
    logging.info("Getting volume info from {}".format(xml))
    # So far, their XML for the volume doesn't play nicely with xml.etree. Thus, we hack.
    paper = etree.Element("paper")
    paper.attrib["id"] = "1000"  # hard-code because there's only one collection.

    volume_text = xml.stem.split(".")[-1]
    title_text = "Transactions of the Association for Computational Linguistics"

    title = etree.Element("title")
    title.text = "{}, Volume {}".format(title_text, volume_text)
    paper.append(title)

    year_text = xml.stem.split(".")[1]
    year = etree.Element("year")
    year.text = year_text
    paper.append(year)

    return paper


def get_paperid(xml: Path, count: int, issue_count: int) -> str:
    basename = xml.stem
    assert int(issue_count) < 10
    assert 0 < count < 1000
    # for i in range(1, 4+1):
    #     assert basename[-i] in [str(x) for x in range(10)], basename
    return f"{issue_count}.{count}"  # after dash in new anth id


def get_title(xml_front_node: etree.Element) -> str:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    title_group = article_meta.find("title-group", nsmap)
    title = title_group.find("article-title", nsmap)
    title_text = collapse_spaces("".join(title.itertext()))
    return title_text


def get_year(xml_front_node: etree.Element) -> str:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    pub_date = article_meta.find("pub-date", nsmap)
    year_text = pub_date.find("year", nsmap).text
    return year_text


def get_month(xml_front_node: etree.Element) -> str:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    pub_date = article_meta.find("pub-date", nsmap)
    try:
        month_id = pub_date.find("month", nsmap).text
    except AttributeError:
        return None
    months = [
        None,
        "January",
        "February",
        "March",
        "April",
        "May",
        "June",
        "July",
        "August",
        "September",
        "October",
        "November",
        "December",
    ]
    month_text = months[int(month_id)]
    return month_text


def get_abstract(xml_front_node: etree.Element) -> str:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    abstract = article_meta.find("abstract", nsmap)
    if abstract is not None:
        abstract_text = collapse_spaces("".join(abstract.itertext()))
        return abstract_text
    else:
        return None


def get_authors(xml_front_node: etree.Element) -> List[Tuple[str, str]]:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    contrib_group = article_meta.find("contrib-group", nsmap)
    authors = []
    for author in contrib_group.findall("contrib", nsmap):
        string_name = author.find("name", nsmap)
        try:
            given_names = string_name.find("given-names", nsmap).text
        except AttributeError:
            given_names = ""  # Special case for Mausam, and potentially Madonna.
        surname = string_name.find("surname", nsmap).text
        try:
            suffix = string_name.find("suffix", nsmap).text
            surname = surname + " " + suffix
        except AttributeError:
            pass
        authors.append((given_names, surname))
    return authors


def get_pages(xml_front_node: etree.Element) -> Tuple[str, str]:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    fpage = article_meta.find("fpage", nsmap)
    lpage = article_meta.find("lpage", nsmap)
    return fpage.text, lpage.text


def get_doi(xml_front_node: etree.Element) -> str:
    nsmap = xml_front_node.nsmap

    article_meta = xml_front_node.find("article-meta", nsmap)
    doi_ = article_meta.find("*[@pub-id-type='doi']", nsmap)
    return doi_.text


def get_article_journal_info(xml_front_node: etree.Element, is_tacl: bool) -> str:
    """ """
    nsmap = xml_front_node.nsmap

    journal_meta = xml_front_node.find("journal-meta", nsmap)

    journal_title_group = journal_meta.find("journal-title-group", nsmap)
    journal_title = journal_title_group.find("journal-title", nsmap)
    # For some reason, sometimes this is not present, so look for this one
    if journal_title is None:
        journal_title = journal_title_group.find("abbrev-journal-title", nsmap)
    journal_title_text = journal_title.text

    article_meta = xml_front_node.find("article-meta", nsmap)
    volume = article_meta.find("volume", nsmap)

    # Fixes
    journal_title_text = " ".join(
        journal_title_text.split()
    )  # Sometimes it's split onto two lines...
    journal_title_text = (
        journal_title_text.replace(  # Somebody in 2018 didn't know our name?
            "Association of Computational Linguistics",
            "Association for Computational Linguistics",
        )
    )
    volume_text = volume.text.lstrip(
        "0"
    )  # Somebody brilliant decided that 2018 would be "06" instead of "6"

    if is_tacl:
        issue_text = None
        string_date_text = None
        format_string = "{journal}, Volume {volume}"
    else:
        issue = article_meta.find("issue", nsmap)
        issue_text = issue.text

        pub_date = article_meta.find("pub-date", nsmap)
        month = pub_date.find("month", nsmap).text
        year = pub_date.find("year", nsmap).text
        string_date_text = f"{month} {year}"
        format_string = "{journal}, Volume {volume}, Issue {issue} - {date}"

    data = dict(
        journal=journal_title_text,
        volume=volume_text,
        issue=issue_text,
        date=string_date_text,
    )
    logging.debug(format_string.format(**data))
    return format_string.format(**data), issue_text


def process_xml(xml: Path, is_tacl: bool) -> Optional[etree.Element]:
    """ """
    logging.info("Reading {}".format(xml))

    tree = etree.parse(open(str(xml)))
    root = tree.getroot()
    front = root.find("front", root.nsmap)

    info, issue = get_article_journal_info(front, is_tacl)

    paper = etree.Element("paper")

    title_text = get_title(front)
    title = etree.Element("title")
    title.text = title_text
    paper.append(title)

    authors = get_authors(front)
    for given_names, surname in authors:
        first = etree.Element("first")
        first.text = given_names

        last = etree.Element("last")
        last.text = surname

        author = etree.Element("author")
        author.append(first)
        author.append(last)

        paper.append(author)

    doi_text = get_doi(front)
    doi = etree.Element("doi")
    doi.text = doi_text
    paper.append(doi)

    abstract_text = get_abstract(front)
    if abstract_text:
        make_simple_element("abstract", abstract_text, parent=paper)

    pages_tuple = get_pages(front)
    pages = etree.Element("pages")
    pages.text = "–".join(pages_tuple)  # en-dash, not hyphen!
    paper.append(pages)

    return paper, info, issue


def issue_info_to_node(
    issue_info: str, year_: str, volume_id: str, is_tacl: bool
) -> etree.Element:
    """Creates the meta block for a new issue / volume"""
    meta = make_simple_element("meta")

    assert int(year_)

    make_simple_element("booktitle", issue_info, parent=meta)
    make_simple_element("publisher", "MIT Press", parent=meta)
    make_simple_element("address", "Cambridge, MA", parent=meta)

    if not is_tacl:
        month_text = issue_info.split()[-2]  # blah blah blah month year
        if not month_text in {
            "January",
            "February",
            "March",
            "April",
            "May",
            "June",
            "July",
            "August",
            "September",
            "October",
            "November",
            "December",
        }:
            logging.error("Unknown month: " + month_text)
        make_simple_element("month", month_text, parent=meta)

    make_simple_element("year", str(year_), parent=meta)

    return meta


def main(args):
    anthology = Anthology(
        importdir=os.path.join(args.anthology_dir, "data"), require_bibkeys=False
    )

    is_tacl = "tacl" in args.root_dir.stem

    venue = TACL if is_tacl else CL  # J for CL, Q for TACL.
    try:
        year = int(args.root_dir.name[-4:])
    except ValueError:
        logging.warning(f"Expected last four chars of {args.root_dir} to be a year")
        sys.exit(-1)

    collection_id = str(year) + "." + venue

    collection_file = os.path.join(
        args.anthology_dir, "data", "xml", f"{collection_id}.xml"
    )
    if os.path.exists(collection_file):
        collection = etree.parse(collection_file).getroot()
    else:
        collection = make_simple_element("collection", attrib={"id": collection_id})

    tacl_glob = "tacl.20*.*/tacl.20*.*.xml"
    # volume_info = get_volume_info(list(args.year_root.glob("*.*.*/*.*.*.xml"))[0])
    # volume.append(volume_info)

    previous_issue_info = None

    papers = []
    for xml in sorted(args.root_dir.glob("xml/*.xml")):
        papernode, issue_info, issue = process_xml(xml, is_tacl)
        if papernode is None or papernode.find("title").text.startswith("Erratum: “"):
            continue

        pdf_path = xml.parent.parent / "assets" / xml.with_suffix(".pdf").name
        if not pdf_path.is_file():
            logging.error(f"Missing pdf for {pdf_path}")
            sys.exit(1)

        papers.append((papernode, pdf_path, issue_info, issue))

        pdf_destination = Path(args.pdfs_dir)
        pdf_destination = pdf_destination / "pdf" / venue
        pdf_destination.mkdir(parents=True, exist_ok=True)

    # MIT Press does assign its IDs in page order, so we have to sort by page
    def sort_papers_by_page(paper_tuple):
        papernode = paper_tuple[0]
        startpage = int(papernode.find("./pages").text.split("–")[0])
        return startpage

    paper_id = 1  # Stupid non-enumerate counter because of "Erratum: " papers interleaved with real ones.
    for papernode, pdf_path, issue_info, issue in sorted(papers, key=sort_papers_by_page):
        issue = issue or "1"
        if issue_info != previous_issue_info:
            # Emit the new volume info before the paper.
            logging.info(f"New issue")
            logging.info(f"{issue_info} vs. {previous_issue_info}")
            previous_issue_info = issue_info

            # Look for node in tree, else create it
            volume_xml = collection.find(f'./volume[@id="{issue}"]')
            if volume_xml is None:
                # xml volume = journal issue
                volume_xml = make_simple_element(
                    "volume", attrib={"id": issue}, parent=collection
                )
                volume_xml.append(
                    issue_info_to_node(issue_info, year, collection_id, is_tacl)
                )
            else:
                for paper in volume_xml.findall(".//paper"):
                    paper_id = max(paper_id, int(paper.attrib["id"]))
                    print(f"Setting paper_id to {paper_id}")
                paper_id += 1

        anth_id = f"{collection_id}-{issue}.{paper_id}"

        # Check if the paper is already present in the volume
        doi_text = papernode.find("./doi").text
        doi_node = collection.xpath(f'.//doi[text()="{doi_text}"]')
        if len(doi_node):
            logging.info(
                f"Skipping existing paper {anth_id}/{doi_text} with title {papernode.find('title').text}"
            )
            continue

        papernode.attrib["id"] = f"{paper_id}"

        destination = pdf_destination / f"{anth_id}.pdf"
        print(f"Copying {pdf_path} to {destination}")
        shutil.copyfile(pdf_path, destination)
        checksum = compute_hash_from_file(pdf_path)

        url_text = anth_id
        url = etree.Element("url")
        url.attrib["hash"] = checksum
        url.text = url_text
        papernode.append(url)

        # Generate bibkey
        volume = Volume.from_xml(
            volume_xml,
            collection_id,
            anthology.venues,
            anthology.sigs,
            anthology.formatter,
        )
        paper = Paper.from_xml(papernode, volume, anthology.formatter)
        bibkey = anthology.pindex.create_bibkey(paper, vidx=anthology.venues)
        make_simple_element("bibkey", bibkey, parent=papernode)

        # Normalize
        for oldnode in papernode:
            normalize(oldnode, informat="latex")
        volume_xml.append(papernode)

        paper_id += 1

    indent(collection)  # from anthology.utils
    et = etree.ElementTree(collection)
    et.write(collection_file, encoding="UTF-8", xml_declaration=True, with_tail=True)


if __name__ == "__main__":
    import sys

    if sys.version_info < (3, 6):
        sys.stderr.write("Python >=3.6 required.\n")
        sys.exit(1)

    import argparse

    parser = argparse.ArgumentParser(description=__doc__)
    anthology_path = os.path.join(os.path.dirname(sys.argv[0]), "..")
    parser.add_argument(
        "--anthology-dir",
        "-r",
        default=anthology_path,
        help="Root path of ACL Anthology Github repo. Default: %(default)s.",
    )
    pdfs_path = os.path.join(os.environ["HOME"], "anthology-files")
    parser.add_argument(
        "--pdfs-dir",
        "-p",
        default=pdfs_path,
        help="Root path for placement of PDF files",
    )

    verbosity = parser.add_mutually_exclusive_group()
    verbosity.add_argument(
        "-v", "--verbose", action="store_const", const=logging.DEBUG, default=logging.INFO
    )
    verbosity.add_argument(
        "-q", "--quiet", dest="verbose", action="store_const", const=logging.WARNING
    )

    parser.add_argument("--version", action="version", version=f"%(prog)s v{__version__}")
    parser.add_argument("root_dir", metavar="FOLDER", type=Path)

    args = parser.parse_args()
    args.root_dir = args.root_dir.resolve()  # Get absolute path.

    logging.basicConfig(level=args.verbose)

    main(args)


# bin/ingest_mtarchive.py

#!/usr/bin/env python3
"""
Takes a conference TSV file and, optionally, a conference metadata
file, and creates the Anthology XML in the ACL Anthology repository.
This file can then be added to the repo and committed.

Example usage:

- First, fork [the ACL Anthology](https://github.com/acl-org/acl-anthology) to your Github account, and clone it to your drive
- Grab one of the [conference TSV files](https://drive.google.com/drive/u/0/folders/1hC7FDlsXWVM2HSYgdluz01yotdEd0zW8)
- Export the [conference list file](https://docs.google.com/spreadsheets/d/1fpxmdV_BPwR6BQHyU9VJQxXeSOmy4__5nQCHBEviyAw/edit#gid=0) to conference-meta.tsv

Then run it as

    scripts/ingest_tsv.py [--anthology /path/to/anthology] eamt/eamt.1997.tsv conference-metadata.tsv

this will create a file `/path/to/anthology/data/xml/1997.eamt.xml`.
You can then commit this to your Anthology repo, push to your Github, and create a PR.

This was used for ingesting many of the conferences in the MT Archive.

Author: Matt Post
March 2020
"""

import csv
import lxml.etree as etree
import os
import shutil
import ssl
import subprocess
import sys
import urllib.request

from anthology.utils import (
    make_simple_element,
    indent,
    compute_hash_from_file,
    retrieve_url,
)
from datetime import datetime
from normalize_anth import normalize


def extract_pages(source_path, page_range, local_path):
    if os.path.exists(local_path):
        print(f"{local_path} already exists, not re-extracting", file=sys.stderr)
        return True
    if not os.path.exists(source_path):
        print(f"{source_path} does not exists", file=sys.stderr)
        raise Exception(f"Could not extract pdf")
    try:
        if "--" in page_range:
            page_range = page_range.replace("--", "-")
        page_range = ' A'.join(page_range.split(','))
        print(
            f"-> Extracting pages {page_range} from {source_path} to {local_path}",
            file=sys.stderr,
        )
        command = [f"pdftk A={source_path} cat {page_range} output {local_path}"]
        print(command, file=sys.stderr)
        subprocess.check_call(command, shell=True)
    except ssl.SSLError:
        raise Exception(f"Could not extract pdf")

    return True


def main(args):
    year, venue, _ = os.path.basename(args.tsv_file.name).split(".")

    # Set the volume name from the collection file, or default to 1
    # The file name is either "2012.eamt.tsv" or "2012.eamt-main.tsv".
    # The default volume name is "1".
    if "-" in venue:
        venue, volume_id = venue.split("-")
    else:
        volume_id = "1"

    collection_id = f"{year}.{venue}"

    tree = etree.ElementTree(
        make_simple_element("collection", attrib={"id": collection_id})
    )

    now = datetime.now()
    today = f"{now.year}-{now.month:02d}-{now.day:02d}"

    volume = make_simple_element("volume", attrib={"id": volume_id, "ingest-date": today})
    tree.getroot().insert(0, volume)

    # Location of entire-proceedings PDF
    proceedings_pdf = args.proceedings

    # Create the metadata for the paper
    meta = None
    for row in csv.DictReader(args.meta_file, delimiter=args.delimiter):
        current_collection_id = f"{row['Year']}.{row['Conference code']}"
        if current_collection_id == collection_id:
            meta = make_simple_element("meta", parent=volume)
            make_simple_element("booktitle", row["Conference title"], parent=meta)
            make_simple_element("publisher", row["Publisher"], parent=meta)
            make_simple_element("address", row["Location"], parent=meta)
            make_simple_element("month", row["Dates held"], parent=meta)
            make_simple_element("year", row["Year"], parent=meta)

            url = row["URL"]

            if url.endswith(".pdf"):
                if proceedings_pdf:
                    print(
                        "Overriding --proceedings with proceedings PDF found in conference list",
                        file=sys.stderr,
                    )
                proceedings_pdf = url

            elif "Complete PDF" in row and row["Complete PDF"] != "":
                proceedings_pdf = row["Complete PDF"]

            # volume PDF
            if proceedings_pdf is not None:
                volume_anth_id = f"{collection_id}-{volume_id}"
                pdf_local_path = os.path.join(
                    args.anthology_files_path, venue, f"{volume_anth_id}.pdf"
                )
                retrieve_url(proceedings_pdf, pdf_local_path)
                checksum = compute_hash_from_file(pdf_local_path)
                make_simple_element(
                    "url", volume_anth_id, attrib={"hash": checksum}, parent=meta
                )
                proceedings_pdf = pdf_local_path

            if row["Editors"] != "" and "?" not in row["Editors"]:
                editors = row["Editors"].split(" and ")
                for editor_name in editors:
                    editor = make_simple_element("editor", parent=meta)
                    if ", " in editor_name:
                        last, first = editor_name.split(", ")
                    else:
                        first, last = (
                            ' '.join(editor_name.split()[:-1]),
                            editor_name.split()[-1],
                        )
                    make_simple_element("first", first, parent=editor)
                    make_simple_element("last", last, parent=editor)
            break
    else:
        print(
            f"Couldn't find conference code {collection_id} in 'Conference code' field of metadata file {args.meta_file.name}",
            file=sys.stderr,
        )
        sys.exit(1)

    paperid = 0
    # Create entries for all the papers
    for row in csv.DictReader(args.tsv_file, delimiter=args.delimiter):
        pages = row.get("Pagenumbers", None)

        title_text = row["Title"]

        # The first row might be front matter (needs a special name)
        if title_text == "Frontmatter" and paperid == 0:
            paper = make_simple_element("frontmatter", parent=volume)

        else:
            paperid += 1
            paper = make_simple_element(
                "paper", attrib={"id": str(paperid)}, parent=volume
            )
            # Only make the title for not-the-frontmatter
            make_simple_element("title", title_text, parent=paper)

            author_list = row["Authors"].split(" and ")
            for author_name in author_list:
                if author_name == "":
                    continue
                author = make_simple_element("author", parent=paper)
                if ", " in author_name:
                    last, first = author_name.split(", ")
                else:
                    first, last = (
                        ' '.join(author_name.split()[:-1]),
                        author_name.split()[-1],
                    )
                make_simple_element("first", first, parent=author)
                make_simple_element("last", last, parent=author)

        if pages is not None:
            make_simple_element("pages", pages, parent=paper)

        # Find the PDF, either listed directly, or extracted from the proceedings PDF
        anth_id = f"{collection_id}-{volume_id}.{paperid}"
        pdf_local_path = os.path.join(args.anthology_files_path, venue, f"{anth_id}.pdf")
        url = None
        if "Pdf" in row and row["Pdf"] != "":
            if retrieve_url(row["Pdf"], pdf_local_path):
                url = anth_id

        elif "pages in pdf" in row:
            pdf_pages = row["pages in pdf"]
            extract_pages(proceedings_pdf, pdf_pages, pdf_local_path)
            url = anth_id

        if url is not None:
            checksum = compute_hash_from_file(pdf_local_path)

            make_simple_element("url", url, attrib={"hash": checksum}, parent=paper)

        if "Abstract" in row:
            make_simple_element("abstract", row["Abstract"], parent=paper)

        if "Presentation" in row:
            url = row["Presentation"]
            if url is not None and url != "" and url != "None":
                extension = row["Presentation"].split(".")[-1]
                name = f"{anth_id}.Presentation.{extension}"
                local_path = os.path.join(
                    args.anthology_files_path,
                    "..",
                    "attachments",
                    venue,
                    name,
                )
                if retrieve_url(row["Presentation"], local_path):
                    checksum = compute_hash_from_file(local_path)
                    make_simple_element(
                        "attachment",
                        name,
                        attrib={"type": "presentation", "hash": checksum},
                        parent=paper,
                    )

        # Normalize
        for node in paper:
            normalize(node, informat="latex")

    indent(tree.getroot())

    # Write the file to disk: acl-anthology/data/xml/{collection_id}.xml
    collection_file = os.path.join(args.anthology, "data", "xml", f"{collection_id}.xml")
    tree.write(collection_file, encoding="UTF-8", xml_declaration=True, with_tail=True)


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('tsv_file', type=argparse.FileType("r"))
    parser.add_argument(
        'meta_file', type=argparse.FileType("r"), help="Path to conference metadata file"
    )
    parser.add_argument(
        '--anthology',
        default=f"{os.environ.get('HOME')}/code/acl-anthology",
        help="Path to Anthology repo (cloned from https://github.com/acl-org/acl-anthology)",
    )
    parser.add_argument(
        '--anthology-files-path',
        default=f"{os.environ.get('HOME')}/anthology-files/pdf",
        help="Path to Anthology files (Default: ~/anthology-files",
    )
    parser.add_argument(
        "--delimiter", "-d", default="\t", help="CSV file delimiter (default: TAB)"
    )
    parser.add_argument('--proceedings', help="Path to PDF with conference proceedings")
    parser.add_argument('--frontmatter', action="store_true")
    parser.add_argument("--force", "-f", action="store_true")
    args = parser.parse_args()

    main(args)


# bin/ingest_pwc.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""
Used to import the links to code and data from Papers with Code (paperswithcode.com)
"""

import logging
import json
import lxml.etree as etree
import os
import logging as log
import requests
import sys


def format_str(x):
    """Format as string if a value is missing or bool."""
    if x is None:
        return ""
    elif isinstance(x, bool):
        return "true" if x else "false"
    else:
        return str(x)


def shift_tails(element):
    """Shift XML children tails to preserve the exact formatting"""
    children = list(element)
    children[-1].tail = children[-2].tail
    children[-2].tail = children[-3].tail


def remove_and_shift_tails(element, child):
    """Remove element and make tails consistent"""
    children = list(element)
    inx = children.index(child)

    if inx > 0:
        children[inx - 1].tail = children[inx].tail

    element.remove(child)


if __name__ == "__main__":
    import argparse

    ap = argparse.ArgumentParser(description="Import metadata from Papers with Code")
    ap.add_argument(
        "-i", "--infile", help="Input metadata JSON (default: fetch from PWC API)"
    )
    args = ap.parse_args()

    logging.basicConfig(level=logging.INFO)

    if args.infile:
        with open(args.infile, "r") as f:
            pwc_meta = json.load(f)
    else:
        res = requests.get("https://paperswithcode.com/integrations/acl")
        if res.ok:
            pwc_meta = res.json()
        else:
            log.warning("Couldn't fetch metadata from Papers with Code (server error).")
            sys.exit(1)

    data_base = "data/xml"

    for xml_filename in os.listdir(data_base):
        # skip any non-xml files
        if not xml_filename.endswith(".xml"):
            continue

        full_path = os.path.join(data_base, xml_filename)

        # load
        with open(full_path) as f:
            tree = etree.parse(f)

        # track if we modified
        old_content = etree.tostring(
            tree, encoding="UTF-8", xml_declaration=True, with_tail=True
        ).decode("utf8")

        for volume in tree.findall("volume"):
            for paper in volume.findall("paper"):
                acl_url = paper.find("url")
                if acl_url is not None:
                    acl_id = acl_url.text
                else:
                    # skip if we cannot construct the id
                    continue

                # start by removing any old entries
                for old in paper.findall("pwccode"):
                    remove_and_shift_tails(paper, old)
                for old in paper.findall("pwcdataset"):
                    remove_and_shift_tails(paper, old)

                if acl_id in pwc_meta:
                    pwc = pwc_meta[acl_id]
                    pwc_code = pwc["code"]
                    if pwc_code["url"] or pwc_code["additional"]:
                        code = etree.SubElement(paper, "pwccode")
                        code.set("url", format_str(pwc_code["url"]))
                        code.set("additional", format_str(pwc_code["additional"]))
                        if pwc_code["name"]:
                            code.text = pwc_code["name"]
                        shift_tails(paper)

                    for pwc_data in pwc["datasets"]:
                        data = etree.SubElement(paper, "pwcdataset")
                        data.set("url", pwc_data["url"])
                        data.text = pwc_data["name"]
                        shift_tails(paper)

        new_content = etree.tostring(
            tree, encoding="UTF-8", xml_declaration=True, with_tail=True
        ).decode("utf8")

        if old_content != new_content:
            with open(full_path, "w") as outfile:
                outfile.write(new_content + "\n")  # all files end with newline

            log.info(f"Modified Papers with Code metadata in {full_path}")


# bin/ingest_tsv.py

#!/usr/bin/env python3
"""
Takes a conference TSV file and creates the Anthology XML in the ACL Anthology repository.
This file can then be added to the repo and committed.

Expects fields name like BibTeX files.

Example usage:

    cat data.tsv \
    | ./ingest_tsv.py amta user 2020 --proceedings-pdf 2020.amta-user.pdf

where data.tsv has TSV fields:

* author
* title
* booktitle
* month
* year
* address
* publisher
* pages
* pdf

Author: Matt Post
October 2020
"""

import anthology
import csv
import lxml.etree as etree
import os
import shutil
import ssl
import subprocess
import sys
import urllib.request

from anthology import Anthology
from anthology.utils import (
    make_simple_element,
    indent,
    compute_hash_from_file,
    retrieve_url,
)
from datetime import datetime
from normalize_anth import normalize
from likely_name_split import NameSplitter
from ingest import maybe_copy
from ingest_mtarchive import extract_pages


def main(args):
    year = args.year
    venue = args.venue
    volume_id = args.volume
    collection_id = f"{year}.{venue}"

    splitter = NameSplitter(anthology_dir=args.anthology_dir)

    collection_file = os.path.join(
        args.anthology_dir, "data", "xml", f"{collection_id}.xml"
    )
    if os.path.exists(collection_file):
        tree = etree.parse(collection_file)
    else:
        tree = etree.ElementTree(
            make_simple_element("collection", attrib={"id": collection_id})
        )

    now = datetime.now()
    today = f"{now.year}-{now.month:02d}-{now.day:02d}"

    volume_node = tree.getroot().find(f"./volume[@id='{volume_id}']")
    if volume_node is not None:
        tree.getroot().remove(volume_node)

    volume = make_simple_element(
        "volume", attrib={"id": volume_id, "ingest-date": today}, parent=tree.getroot()
    )

    if not os.path.exists(collection_id):
        print(f"Creating {collection_id}", file=sys.stderr)
        os.makedirs(collection_id)

    # Create entries for all the papers
    for paperid, row in enumerate(
        csv.DictReader(args.tsv_file, delimiter=args.delimiter)
    ):
        pages = row.get("pages", None)

        if paperid == 0:
            meta = make_simple_element("meta", parent=volume)
            make_simple_element("booktitle", row["booktitle"], parent=meta)
            make_simple_element("publisher", row["publisher"], parent=meta)
            make_simple_element("address", row["address"], parent=meta)
            make_simple_element("month", row["month"], parent=meta)
            make_simple_element("year", year, parent=meta)

            editors = row["author"].split(" and ")
            row["author"] = ""
            for editor_name in editors:
                editor = make_simple_element("editor", parent=meta)
                surname, givenname = splitter.best_split(editor_name)
                make_simple_element("first", givenname, parent=editor)
                make_simple_element("last", surname, parent=editor)

            # volume PDF
            proceedings_pdf = args.proceedings_pdf
            if proceedings_pdf is not None:
                volume_anth_id = f"{collection_id}-{volume_id}"
                pdf_local_path = os.path.join(
                    args.anthology_files_path, venue, f"{volume_anth_id}.pdf"
                )
                retrieve_url(proceedings_pdf, pdf_local_path)
                checksum = compute_hash_from_file(pdf_local_path)
                make_simple_element(
                    "url", volume_anth_id, attrib={"hash": checksum}, parent=meta
                )
                proceedings_pdf = pdf_local_path

        title_text = row["title"]

        # The first row might be front matter (needs a special name)
        if paperid == 0 and title_text.lower() in ["frontmatter", "front matter"]:
            paper = make_simple_element("frontmatter", parent=volume)
        else:
            if paperid == 0:
                # Not frontmatter, so paper 1
                paperid += 1

            paper = make_simple_element(
                "paper", attrib={"id": str(paperid)}, parent=volume
            )
            # Only make the title for not-the-frontmatter
            make_simple_element("title", title_text, parent=paper)

        author_list = row["author"].split(" and ")

        for author_name in author_list:
            if author_name == "":
                continue
            author = make_simple_element("author", parent=paper)
            surname, givenname = splitter.best_split(author_name)
            make_simple_element("first", givenname, parent=author)
            make_simple_element("last", surname, parent=author)

        if pages is not None and pages != "":
            make_simple_element("pages", pages, parent=paper)

        # Find the PDF, either listed directly, or extracted from the proceedings PDF
        anth_id = f"{collection_id}-{volume_id}.{paperid}"
        pdf_local_path = os.path.join(args.anthology_files_path, venue, f"{anth_id}.pdf")
        url = None
        if "pdf" in row and row["pdf"] != "":
            if retrieve_url(row["pdf"], pdf_local_path):
                url = anth_id
            else:
                print("Can't find", row["pdf"])

        elif "pages in pdf" in row:
            pdf_pages = row["pages"]
            extract_pages(proceedings_pdf, pdf_pages, pdf_local_path)
            url = anth_id

        if url is not None:
            checksum = compute_hash_from_file(pdf_local_path)

            make_simple_element("url", url, attrib={"hash": checksum}, parent=paper)

        if "abstract" in row and row["abstract"] != "":
            make_simple_element("abstract", row["abstract"], parent=paper)

        if "presentation" in row:
            url = row["presentation"]
            if url is not None and url != "" and url != "None":
                extension = row["presentation"].split(".")[-1]
                name = f"{anth_id}.Presentation.{extension}"
                local_path = os.path.join(
                    args.anthology_files_path,
                    "..",
                    "attachments",
                    venue,
                    name,
                )
                if retrieve_url(row["presentation"], local_path):
                    make_simple_element(
                        "attachment",
                        name,
                        attrib={
                            "type": "presentation",
                            "hash": compute_hash_from_file(local_path),
                        },
                        parent=paper,
                    )

        # Normalize
        for node in paper:
            normalize(node, informat="latex")

    indent(tree.getroot())

    # Write the file to disk: acl-anthology/data/xml/{collection_id}.xml
    tree.write(collection_file, encoding="UTF-8", xml_declaration=True, with_tail=True)


if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--tsv-file', nargs="?", default=sys.stdin, type=argparse.FileType("r")
    )
    parser.add_argument(
        '--anthology-dir',
        default=f"{os.environ.get('HOME')}/code/acl-anthology",
        help="Path to Anthology repo (cloned from https://github.com/acl-org/acl-anthology)",
    )
    parser.add_argument(
        '--anthology-files-path',
        default=f"{os.environ.get('HOME')}/anthology-files/pdf",
        help="Path to Anthology files (Default: ~/anthology-files",
    )
    parser.add_argument(
        "--delimiter", "-d", default="\t", help="CSV file delimiter (default: TAB)"
    )
    parser.add_argument(
        '--proceedings-pdf', help="Path to PDF with conference proceedings"
    )
    parser.add_argument("venue", help="Venue code, e.g., acl")
    parser.add_argument("volume", help="Volume name, e.g., main or 1")
    parser.add_argument("year", help="Full year, e.g., 2020")
    args = parser.parse_args()

    main(args)


# bin/latex_to_unicode.py

#!/usr/bin/env python3

import sys
import logging
import collections, copy
import re
import latexcodec, codecs
import lxml.etree as etree

Entry = collections.namedtuple("Entry", ["open", "close", "tag", "type", "verbatim"])
table = [
    Entry("{", "}", None, "bracket", False),
    Entry("$", "$", "tex-math", "bracket", True),
    Entry(r"\(", r"\)", "tex-math", "bracket", True),
    Entry(r"\emph", None, "i", "unary", False),
    Entry(r"\em", None, "i", "setter", False),
    Entry(r"\textit", None, "i", "unary", False),
    Entry(r"\it", None, "i", "setter", False),
    Entry(r"\textsl", None, "i", "unary", False),
    Entry(r"\sl", None, "i", "setter", False),
    Entry(r"\textbf", None, "b", "unary", False),
    Entry(r"\bf", None, "b", "setter", False),
    # Entry(r'\textsc', None, 'sc', 'unary', False),
    # Entry(r'\sc', None, 'sc', 'setter', False),
    Entry(r"\url", None, "url", "unary", True),
    Entry(r"\footnote", None, "fn", "unary", False),
    Entry(r"\href", None, "delete", "unary", False),
    # \cite and friends. There are a lot more, but these are the most common.
    Entry(r"\cite", None, "cite", "unary", False),
    Entry(r"\newcite", None, "cite", "unary", False),
    Entry(r"\citet", None, "cite", "unary", False),
    Entry(r"\citep", None, "cite", "unary", False),
    Entry(r"\fixedcase", None, "fixed-case", "unary", False),  # for our internal use
    Entry(r"", None, None, "trivial", False),
]
openers = {e.open: e for e in table}
closers = {e.close: e for e in table if e.type == "bracket"}
tags = {e.tag: e for e in table}

token_re = re.compile(r"\\[A-Za-z]+\s*|\\.|.", re.DOTALL)


def tokenize_latex(s):
    return token_re.findall(s)


def parse_latex(s):
    """Parse LaTeX into a list of lists. The original string can be
    recovered by recursively joining the lists. A list is formed for
    anything listed in `table`. The first and last element of the list
    are always the opener and (possibly empty) closer, and the middle
    elements are the contents.
    """

    stack = [[""]]

    def close_implicit():
        # Implicitly close setters
        top = stack.pop()
        open = top[0].rstrip()
        if open == "$":
            logging.warning("unmatched $, treating as dollar sign")
            stack[-1].extend(top)
        else:
            if openers[open].type != "setter":
                logging.warning("closing unmatched {}".format(open))
            top.append("")
            stack[-1].append(top)

    math_mode = False
    for tok in tokenize_latex(s):
        tokr = tok.rstrip()

        # An opener starts a new list
        if (
            tokr != "$"
            and tokr in openers
            and openers[tokr].type != "trivial"
            or tokr == "$"
            and not math_mode
        ):
            stack.append([tok])

        # A closer
        elif tokr != "$" and tokr in closers or tokr == "$" and math_mode:

            # Look for the matching opener, which might not be on top
            for i in reversed(range(len(stack))):
                if stack[i][0].rstrip() == closers[tokr].open:
                    break
            else:
                logging.warning("unexpected {}".format(tokr))
                stack[-1].append(tok)
                continue

            # Close any intervening openers (e.g., \em in {a \em b})
            while len(stack) - 1 > i:
                close_implicit()

            stack[-1].append(tok)
            top = stack.pop()
            stack[-1].append(top)

        else:
            stack[-1].append(tok)

        if tokr == "$":
            math_mode = not math_mode

        if len(stack[-1]) == 2:
            open = stack[-1][0].rstrip()
            if openers[open].type == "unary":
                top = stack.pop()
                top.append("")
                stack[-1].append(top)

    while len(stack) > 1:
        close_implicit()
    stack[-1].append("")

    return stack[0]


def flatten(l):
    def visit(l):
        if isinstance(l, str):
            out.append(l)
        else:
            for c in l:
                visit(c)

    out = []
    visit(l)
    return "".join(out)


def latex_to_unicode(s):
    """Convert (La)TeX control sequences in string s to their Unicode equivalents."""

    # Convert \\ to newline; this also ensures that remaining
    # backslashes really introduce control sequences.
    s = re.sub(r"\\\\", "\n\n", s)

    ### Do a few conversions in the reverse direction first

    # % is probably percent (not a comment delimiter)
    s = re.sub(r"(?<!\\)%", r"\%", s)

    # Use a heuristic to decide whether ~ means "approximately" or is a tie
    s = re.sub(r"(?<=[ (])~(?=\d)", r"\\textasciitilde ", s)
    s = re.sub(r"^~(?=\d)", r"\\textasciitilde ", s)

    # An old bug in our system converted --- to –-; this undoes it
    s = s.replace("–", "--")

    ### Some normalization to make later processing easier.

    # It's easier to deal with control sequences if followed by exactly one space.
    # latexcodec does preserve this space.
    s = re.sub(r"(\\[A-Za-z]+)\s*", r"\1 ", s)

    # Change strings like \'{e} into {\'e}.
    # Also, this avoids a latexcodec bug for \"{\i}, etc.
    s = re.sub(r"(\\[A-Za-z]+ |\\.)\{([^\\{}]|\\i )}", r"{\1\2}", s)

    ### Call latexcodec

    # preserve leading space, which latexcodec strips
    leading_space = len(s) > 0 and s[0].isspace()
    s = codecs.decode(s, "ulatex+utf8")
    if leading_space:
        s = " " + s

    ### Missing from latexcodec (as of version 1.0.7)
    s = re.sub(r"\\textcommabelow ([SsTt])", "\\1\N{COMBINING COMMA BELOW}", s)
    s = s.replace(r"\dj ", "đ")
    s = s.replace(r"\DJ ", "Đ")
    s = s.replace(r"\hwithstroke ", "ħ")
    s = s.replace(r"\Hwithstroke ", "Ħ")
    s = s.replace(r"\textquotesingle ", "'")
    s = s.replace(r"\textquotedblleft ", "“")
    s = s.replace(r"\textquotedblright ", "”")

    ### Intentionally missing from latexcodec
    s = s.replace(r"\$", "$")
    s = s.replace(r"\&", "&")
    s = s.replace("`", "‘")

    # Clean up
    s = re.sub(r"(?<!\\)[{}]", "", s)  # unescaped curly braces
    s = s.replace(r"\{", "{")
    s = s.replace(r"\}", "}")

    def repl(s):
        logging.warning(f"discarding control sequence '{s.group(0)}' from '{s.string}'")
        return ""

    s = re.sub(r"\\[A-Za-z]+ |\\.", repl, s)

    return s


math_table = {r"\sim": r"\textasciitilde"}
for c in list(".,@%~") + [r"\%"]:
    math_table[c] = c


def flatten_trivial_math(node):
    """Convert math that doesn't really need to be math into text."""

    def visit(node):
        if isinstance(node, str):
            return
        elif openers[node[0].rstrip()].tag == "tex-math":
            if all(
                isinstance(child, str)
                and (child.isspace() or child.isdigit() or child in math_table)
                for child in node[1:-1]
            ):
                node[0] = "{"
                for i in range(1, len(node) - 1):
                    if node[i] in math_table:
                        node[i] = math_table[node[i]]
                node[-1] = "}"
        for child in node:
            visit(child)

    node = copy.deepcopy(node)
    visit(node)
    return node


def remove_notes(node):
    def visit(node, outparent):
        if isinstance(node, str):
            outparent.append(node)
        elif isinstance(node, list):
            if openers[node[0].rstrip()].tag in ["fn", "delete"]:
                return  # without copying
            elif openers[node[0].rstrip()].tag == "cite":
                outparent.append("(CITATION)")
            else:
                outnode = []
                for child in node:
                    visit(child, outnode)
                outparent.append(outnode)

    outroot = []
    visit(node, outroot)
    assert len(outroot) == 1
    return outroot[0]


def append_text(xnode, text):
    if len(xnode) == 0:
        xnode.text = (xnode.text or "") + text
    else:
        xnode[-1].tail = (xnode[-1].tail or "") + text


def latextree_to_xml(node):
    """Convert a LaTeX tree to XML."""

    def visit(node, xparent=None):
        if isinstance(node, str):
            append_text(xparent, node)

        else:
            open = node[0].rstrip()

            tag = openers[open].tag
            if tag is None:
                # Convert opener and closer into text
                append_text(xparent, node[0])
                for child in node[1:-1]:
                    visit(child, xparent)
                append_text(xparent, node[-1])

            else:
                # Create element with equivalent HTML tag
                xnode = etree.Element(tag)
                xparent.append(xnode)

                if openers[open].type == "unary":
                    # When the argument is a group, the braces are not included.
                    # This is especially important for \url
                    assert len(node) == 3 and node[1][0] == "{"
                    node = node[1]

                if openers[open].verbatim:
                    xnode.text = "".join(flatten(child) for child in node[1:-1])
                else:
                    for child in node[1:-1]:
                        visit(child, xnode)

    xroot = etree.Element("root")
    visit(node, xroot)
    return xroot


def xml_to_unicode(xtree):
    """Call latex_to_unicode on all text nodes in an XML tree."""

    def visit(xnode):
        if xnode.tag == "root" or not tags[xnode.tag].verbatim:
            if xnode.text:
                xnode.text = latex_to_unicode(xnode.text)
            for child in xnode:
                visit(child)
        if xnode.tail:
            xnode.tail = latex_to_unicode(xnode.tail)

    xtree = copy.deepcopy(xtree)
    visit(xtree)
    return xtree


def latex_to_xml(s, fixed_case=False, trivial_math=False):
    tree = parse_latex(s)
    if fixed_case:
        tree = find_fixed_case(tree, conservative=True)
    if trivial_math:
        tree = flatten_trivial_math(tree)
    tree = remove_notes(tree)
    tree = latextree_to_xml(tree)
    tree = xml_to_unicode(tree)
    return tree


def find_fixed_case(node, conservative=False):
    def visit(cur, prev):
        if isinstance(cur, str):
            return cur

        if cur[0] == "{":
            if (
                isinstance(cur[1], str)
                and cur[1].startswith("\\")
                or isinstance(cur[1], list)
                and cur[1][0].startswith("\\")
            ):
                # {\...} does *not* protect case
                return  # and don't recurse
            elif conservative and isinstance(prev, str) and prev.startswith("\\"):
                # \cmd{...} does protect case, but in practice,
                # this never seems to be the intent.
                pass
            elif (
                conservative
                and len(cur) == 3
                and isinstance(cur[1], list)
                and cur[1][0] in ["$", r"\)"]
            ):
                # Don't mark {$...$}
                pass
            else:
                cur[:] = [r"\fixedcase", cur[:], ""]

        elif cur[0] in ["$", r"\)"]:
            return  # Don't recurse into math

        prev = cur[0]
        for child in cur[1:-1]:
            visit(child, prev)
            prev = child

    if conservative:
        # Check if whole field is surrounded with braces
        braces = 0
        text = False
        for child in node[1:-1]:
            if isinstance(child, str) and not child.isspace():
                text = True
            if isinstance(child, list) and child[0] == "{":
                braces += 1
        if not text and braces == 1:
            return node

    node = copy.deepcopy(node)
    visit(node, None)
    return node


if __name__ == "__main__":
    import fileinput

    for line in fileinput.input():
        line = line.rstrip()
        tree = latex_to_xml(line, fixed_case=True, trivial_math=True)
        print(etree.tostring(tree, encoding="unicode"))
        # print(latex_to_unicode(line))


# bin/likely_name_split.py

#!/usr/bin/env python3
# Daniel Gildea, 2020

"""Usage: likely_name_split.py [--importdir=DIR]

Counts first and last names in anthology.
Predicts best split into first and last.
Checks whether current names match our predictions.

Options:
  --importdir=DIR          Directory to import XML files from. [default: {scriptdir}/../data/]
  -h, --help               Display this helpful text.
"""

from collections import defaultdict
from docopt import docopt
import pickle, json
import sys
import re
import os
from math import *

from anthology import Anthology
from anthology.people import PersonName


def log0(x):
    if x == 0:
        return -inf
    else:
        return log(x)


class NameSplitter:
    def __init__(self, anthology=None, anthology_dir=None):
        # counts of how often each name appears
        self.first_count = defaultdict(lambda: 0)  # "Maria" "Victoria"
        self.first_full_count = defaultdict(lambda: 0)  # "Maria Victoria"
        self.last_count = defaultdict(lambda: 0)  # "van" "den" "Bosch"
        self.last_full_count = defaultdict(lambda: 0)  # "van den Bosch"
        self.first_total = 0
        self.last_total = 0

        if os.path.exists("names.cache"):
            self.load_cache()
        else:
            if anthology is None and anthology_dir is not None:
                anthology = Anthology(os.path.join(anthology_dir, "data"))
            self.count_names(anthology)
            self.dump_cache()

    def load_cache(self):
        with open("names.cache", "r") as cache:
            p = json.load(cache)
            self.first_count = defaultdict(int, p["first_count"])
            self.first_full_count = defaultdict(int, p["first_full_count"])
            self.first_total = p["first_total"]
            self.last_count = defaultdict(int, p["last_count"])
            self.last_full_count = defaultdict(int, p["last_full_count"])
            self.last_total = p["last_total"]
        print(f"Loaded cache from names.cache", file=sys.stderr)

    def dump_cache(self):
        with open("names.cache", "w") as cache:
            p = {
                "first_count": self.first_count,
                "first_full_count": self.first_full_count,
                "first_total": self.first_total,
                "last_count": self.last_count,
                "last_full_count": self.last_full_count,
                "last_total": self.last_total,
            }
            print(json.dumps(p), file=cache)
        print(f"Dumped counts to names.cache", file=sys.stderr)

    # counts names in anthology database into global vars
    # first_count last_count (dicts)
    # first_full_count last_full_count (dicts)
    # first_total last_total (floats)
    def count_names(self, anthology):
        for person in anthology.people.personids():
            name = anthology.people.get_canonical_name(person)
            num_papers = len(anthology.people.get_papers(person)) + 0.0
            # print(name.last, ", ", name.first, num_papers)
            for w in name.first.split(" "):
                self.first_count[w] += num_papers
            self.first_full_count[name.first] += num_papers
            self.first_total += num_papers

            for w in name.last.split(" "):
                self.last_count[w] += num_papers
            self.last_full_count[name.last] += num_papers
            self.last_total += num_papers

    # takes "Maria Victoria Lopez Gonzalez"
    # returns ("Lopez Gonzalez", "Maria Victoria")
    # uses counts of words in first and last names in current database
    def best_split(self, name):
        if "," in name and not "Jr." in name:
            # Short-circuit names that are already split
            # comma in "William Baumgartner, Jr." does not count as a split
            surname, given_names = name.split(",")
            return (surname.strip(), given_names.strip())

        words = name.split(" ")
        best_score = -inf
        best = ("", "")
        # loop over possible split points between first/last
        for i in range(1, len(words)):  # at least one word in each part
            first = " ".join(words[0:i])
            last = " ".join(words[i:])
            # max of log prob of "Maria Victoria" and
            # log prob of "Maria" + log prob of "Victoria"
            first_probs = [
                # more smoothing for first than last name,
                # so that default is one-word last name when all counts are zero
                log((self.first_count[x] + 0.1) / self.first_total)
                for x in words[0:i]
            ]
            first_score = max(
                # no smoothing for multiword name: log(0) => -inf
                log0((self.first_full_count[first]) / self.first_total),
                sum(first_probs),
            )
            last_probs = [
                log((self.last_count[x] + 0.01) / self.last_total) for x in words[i:]
            ]
            last_score = max(
                log0((self.last_full_count[last]) / self.last_total), sum(last_probs)
            )

            if first_score + last_score > best_score:
                best_score = first_score + last_score
                best = (last, first)
            # end of loop over split points
        return best


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )

    anthology = Anthology(importdir=args["--importdir"])
    splitter = NameSplitter(anthology)

    # for all names currently in anthology,
    # see if they match what we predict
    for person in anthology.people.personids():
        name = anthology.people.get_canonical_name(person)

        # find our prediction of split
        best = splitter.best_split(name.first + " " + name.last)

        # if current split does not match our prediction
        if not (best[0] == name.last and best[1] == name.first):
            # print suggested replacement
            print(name.last, ",", name.first, "  ==>  ", best[0], ",", best[1])


# bin/merge_xml.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
# Copyright 2019 David Wei Chiang <dchiang@nd.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


"""
Used to merge in missing information to many papers in commit 24ab9efd2ec05b9dcc80337695e5b33219aab679.
(See issue #173).

Author: David Chiang
"""

import logging
import difflib
import lxml.etree as etree
import copy

unique_attributes = {"id"}
recurse_elements = {"volume", "paper"}
exclude_elements = {
    "paper"
}  # papers are usually removed for a reason; don't put them back


def opentag(node):
    copy = etree.Element(node.tag)
    copy.text = "\n"
    for a, v in sorted(node.attrib.items()):
        if a in unique_attributes:
            copy.attrib[a] = v
    s = etree.tostring(copy, encoding=str)
    return s.splitlines()[0]


def merge(atree, btree):
    def visit(anode, bnode):

        if anode.tag not in recurse_elements and bnode.tag not in recurse_elements:
            return

        achildren = sorted(anode, key=opentag)
        bchildren = sorted(bnode, key=opentag)

        indent = anode.text or ""
        after = anode[-1].tail or ""
        assert indent.strip() == after.strip() == ""

        for op, ai, aj, bi, bj in difflib.SequenceMatcher(
            a=list(map(opentag, achildren)), b=list(map(opentag, bchildren))
        ).get_opcodes():
            if op in ["insert", "replace"]:
                for bchild in bchildren[bi:bj]:
                    if bchild.tag in exclude_elements:
                        logging.info(
                            "don't insert {}:<{}>".format(bchild.sourceline, bchild.tag)
                        )
                        continue
                    # hacky exception: papers shouldn't have editors
                    if bchild.tag == "editor" and bchild.getparent().find(
                        "bibtype"
                    ).text in ["inproceedings", "incollection", "article"]:
                        logging.info(
                            "don't insert {}:<{}>".format(bchild.sourceline, bchild.tag)
                        )
                        continue

                    logging.info(
                        "insert {}:{}".format(
                            bchild.sourceline,
                            etree.tostring(bchild, encoding=str, with_tail=False),
                        )
                    )
                    anode[-1].tail = indent
                    anode.append(copy.deepcopy(bchild))
                    anode[-1].tail = after
            elif op == "equal":
                for achild, bchild in zip(achildren[ai:aj], bchildren[bi:bj]):
                    visit(achild, bchild)

    aroot = atree.getroot()
    broot = btree.getroot()
    visit(aroot, broot)


if __name__ == "__main__":
    import sys
    import argparse

    ap = argparse.ArgumentParser(description="Merge two XML files.")
    ap.add_argument("afile", help="first XML file (fields in this file take priority")
    ap.add_argument("bfile", help="second XML file")
    ap.add_argument("-o", "--outfile", help="XML file to write (default stdout)")
    args = ap.parse_args()

    logging.basicConfig(level=logging.INFO)

    if args.outfile:
        outfile = open(args.outfile, "w")
    else:
        outfile = sys.stdout

    atree = etree.parse(args.afile)
    btree = etree.parse(args.bfile)

    merge(atree, btree)

    outfile.write(
        etree.tostring(
            atree, encoding="UTF-8", xml_declaration=True, with_tail=True
        ).decode("utf8")
    )


# bin/new-authors.py

#!/usr/bin/env python3

"""
Takes a list of collection IDs as arguments, and outputs a TSV
(name, Anthology ID, paper title) containing every person who
is the first author of a paper and has no other papers in the
Anthology.

Place in acl-anthology/bin and run

   ./bin/new-authors.py 2020.acl

which returns all first authors who had their first paper at ACL
2020. Note that this doesn't ignore future volumes, so if run in
2024, this will no longer work for 2020.

Author: Matt Post
"""

import os
import sys

from anthology import Anthology
from anthology.people import PersonName
from anthology.utils import deconstruct_anthology_id

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("collections", nargs="+")
    args = parser.parse_args()

    anthology = Anthology(
        importdir=os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
    )

    # header
    print("name", "id", "title", sep="\t")

    for id_, paper in anthology.papers.items():
        collection_id, volume_name, paper_id = deconstruct_anthology_id(id_)
        if collection_id in args.collections:
            authors = paper.attrib.get("author", [])
            if len(authors) > 0:
                # "authors" is a list of ("last name || first name", name-id or None) tuples
                first_author = authors[0][0]
                authors_papers = list(
                    anthology.people.name_to_papers[first_author].values()
                )
                authors_papers = authors_papers[0] + authors_papers[1]
                if len(authors_papers) == 1:
                    print(first_author.full, id_, paper.get_title('text'), sep="\t")


# bin/normalize_anth.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 David Wei Chiang <dchiang@nd.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Try to convert ACL Anthology XML format to a standard form, in
which:
- All single and double quotes are curly
- Some characters (e.g., fullwidth chars) are mapped to standard equivalents
- Combining accents and characters are composed into single characters
- Letters that are capital "by nature" are protected with <fixed-case>
- With the `-t` option:
  - Outside of formulas, no LaTeX is used; only Unicode
  - Formulas are tagged as <tex-math> and use LaTeX

Usage: python3 normalize_anth.py [-t] <infile> <outfile>

Bugs:

- Doesn't preserve line breaks and indentation within text fields.
"""

import lxml.etree as etree
import re
import difflib
import logging
import unicodedata
import html
from latex_to_unicode import latex_to_xml
from fixedcase.protect import protect

location = ""
logging.basicConfig(format="%(levelname)s:%(location)s %(message)s", level=logging.INFO)


def filter(r):
    r.location = location
    return True


logging.getLogger().addFilter(filter)


def replace_node(old, new):
    save_tail = old.tail
    old.clear()
    old.tag = new.tag
    old.attrib.update(new.attrib)
    old.text = new.text
    old.extend(new)
    old.tail = save_tail


def maptext(node, f):
    if node.tag in ["tex-math", "url"]:
        return
    if node.text is not None:
        node.text = f(node.text)
    for child in node:
        maptext(child, f)
        if child.tail is not None:
            child.tail = f(child.tail)


def curly_quotes(s):
    # Two single quotes after a word: LaTeX for a right curly quote.
    s = re.sub(r'(\w[^\s"]*)\'\'', r"\1”", s)
    # Two single quotes (or backticks) before a word: LaTeX for a left curly quote.
    s = re.sub(r"(``|\'\')(\w)", r"“\2", s)

    # Straight double quote: If preceded by a word (possibly with
    # intervening punctuation), it's a right quote.
    s = re.sub(r'(\w[^\s"]*)"', r"\1”", s)
    # Else, if followed by a word, it's a left quote
    s = re.sub(r'"(\w)', r"“\1", s)
    if '"' in s:
        logging.warning(f"couldn't convert straight double quote in [{s}]")

    # Straight single quote
    # Exceptions for words that start with apostrophe
    s = re.sub(
        r"'(em|round|n|tis|twas|til|cause|scuse|\d0)\b", r"’\1", s, flags=re.IGNORECASE
    )
    # Otherwise, treat the same as straight double quote
    s = re.sub(r"(\w[^\s']*)'", r"\1’", s)
    s = re.sub(r"'(\w)", r"‘\1", s)
    if "'" in s:
        logging.warning(f"couldn't convert straight single quote in [{s}]")

    return s


def clean_unicode(s):
    s = s.replace("\u00ad", "")  # soft hyphen
    s = s.replace("\u2010", "-")  # hyphen

    # Some sources encode an i with an accent above using dotless i,
    # which must be converted to normal i
    s = list(s)
    for i in range(len(s) - 1):
        # bug: we should only be looking for accents above, not
        # below
        if s[i] == "ı" and unicodedata.category(s[i + 1]) == "Mn":
            s[i] = "i"
    s = "".join(s)

    # Selectively apply compatibility decomposition.
    # This converts, e.g., ﬁ to fi and ： to :, but not ² to 2.
    # Unsure: … to ...
    # More classes could be added here.
    def decompose(c):
        d = unicodedata.decomposition(c)
        if d and d.split(None, 1)[0] in ["<compat>", "<wide>", "<narrow>", "<noBreak>"]:
            return unicodedata.normalize("NFKD", c)
        else:
            return c

    s = "".join(map(decompose, s))

    # Convert combining characters when possible
    s = unicodedata.normalize("NFC", s)

    return s


def normalize(oldnode, informat):
    """
    Receives an XML 'paper' node and normalizes many of its fields, including:
    - Unescaping HTML
    - Normalizing quotes and other punctuation
    - Mapping many characters to unicode
    In addition, if the 'informat' is "latex", it will convert many LaTeX characters
    to unicode equivalents. Note that these latter LaTeX operations are not idempotent.
    """

    if oldnode.tag in [
        "url",
        "href",
        "mrf",
        "doi",
        "bibtype",
        "bibkey",
        "revision",
        "erratum",
        "attachment",
        "paper",
        "presentation",
        "dataset",
        "software",
        "video",
    ]:
        return
    elif oldnode.tag in ["author", "editor"]:
        for oldchild in oldnode:
            normalize(oldchild, informat=informat)
    else:
        if informat == "latex":
            if len(oldnode) > 0:
                logging.error(
                    "field has child elements {}".format(
                        ", ".join(child.tag for child in oldnode)
                    )
                )
            oldtext = "".join(oldnode.itertext())
            newnode = latex_to_xml(
                oldtext,
                trivial_math=True,
                fixed_case=oldnode.tag in ["title", "booktitle"],
            )
            newnode.tag = oldnode.tag
            newnode.attrib.update(oldnode.attrib)
            replace_node(oldnode, newnode)

        maptext(oldnode, html.unescape)
        maptext(oldnode, curly_quotes)
        maptext(oldnode, clean_unicode)
        if oldnode.tag in ["title", "booktitle"]:
            protect(oldnode)


if __name__ == "__main__":
    import sys
    import argparse

    ap = argparse.ArgumentParser(description="Convert Anthology XML to standard format.")
    ap.add_argument("infile", help="XML file to read")
    ap.add_argument("outfile", help="XML file to write")
    ap.add_argument(
        "-t",
        "--latex",
        action="store_true",
        help="Assume input fields are in LaTeX (not idempotent",
    )
    args = ap.parse_args()

    if args.latex:
        informat = "latex"
    else:
        informat = "xml"

    tree = etree.parse(args.infile)
    root = tree.getroot()
    if not root.tail:
        # lxml drops trailing newline
        root.tail = "\n"
    for paper in root.findall(".//paper"):
        papernum = "{} vol {} paper {}".format(
            root.attrib["id"], paper.getparent().attrib["id"], paper.attrib["id"]
        )
        for oldnode in paper:
            location = "{}:{}".format(papernum, oldnode.tag)
            normalize(oldnode, informat=informat)

    tree.write(args.outfile, encoding="UTF-8", xml_declaration=True)


# bin/patch_fixedcase.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 David Wei Chiang <dchiang@nd.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import lxml.etree as etree, html
import sys
import os
import logging
import bibtex
import tex_unicode


def replace_node(old, new):
    save_tail = old.tail
    old.clear()
    old.tag = new.tag
    old.attrib.update(new.attrib)
    old.text = new.text
    old.extend(new)
    old.tail = save_tail


if __name__ == "__main__":
    # Set up logging
    logging.basicConfig(
        format="%(levelname)s:%(location)s %(message)s", level=logging.INFO
    )
    location = ""

    def filter(r):
        r.location = location
        return True

    logging.getLogger().addFilter(filter)

    xfilename, bdirname, outfilename = sys.argv[1:]
    xtree = etree.parse(xfilename)
    xroot = xtree.getroot()

    for xpaper in xroot.findall("paper"):
        fullid = "{}-{}".format(xroot.attrib["id"], xpaper.attrib["id"])
        location = fullid
        bfilename = os.path.join(bdirname, fullid + ".bib")
        try:
            bdata = bibtex.read_bibtex(bfilename)
        except FileNotFoundError:
            logging.warning("{} not found".format(bfilename))
            continue
        for entry in bdata.entries.values():
            for field in ["title", "booktitle"]:
                if field not in entry.fields:
                    continue

                boldvalue = entry.fields[field]
                bnewvalue = tex_unicode.parse_latex(boldvalue)
                bibtex.find_fixed_case(bnewvalue, conservative=True)
                bnewvalue = tex_unicode.unparse_latex(bnewvalue, delete_root=True)
                if bnewvalue != boldvalue:
                    xnode = xpaper.find(field)
                    if xnode is None:
                        logging.warning("{} missing from XML".format(field))
                        continue

                    xoldvalue = tex_unicode.contents(xnode)
                    boldvalue = tex_unicode.convert_string(boldvalue)
                    xoldvalue = " ".join(xoldvalue.split())
                    boldvalue = " ".join(boldvalue.split())
                    if xoldvalue != boldvalue:
                        logging.warning(
                            "{} has been changed; please edit manually".format(field)
                        )
                        logging.warning("old value in xml: {}".format(xoldvalue))
                        logging.warning("old value in bib: {}".format(boldvalue))
                        logging.warning("new value in bib: {}".format(bnewvalue))
                    else:  # success
                        bnewvalue = html.escape(bnewvalue)
                        bnewvalue = bnewvalue.replace(
                            r"\begin{fixedcase}", "<fixed-case>"
                        )
                        bnewvalue = bnewvalue.replace(r"\end{fixedcase}", "</fixed-case>")
                        xnewnode = tex_unicode.convert_node(
                            etree.fromstring(
                                "<{}>{}</{}>".format(xnode.tag, bnewvalue, xnode.tag)
                            )
                        )
                        replace_node(xnode, xnewnode)

    xtree.write(outfilename, encoding="UTF-8", xml_declaration=True)


# bin/repair_url.py

"""Repairs URLs in Anthology XML files.

Usage: repair_url.py <infilename> <outfilename>

To do:
- check URLs of attachments
- incorporate into a more general XML-fixing script

"""

import lxml.etree as etree
import requests
import sys

from anthology.utils import test_url


def get_anth_url(volume_id, paper_id=None, width=4):
    return "https://aclanthology.org/{volume_id}-{paper_id:0{width}d}".format(
        volume_id=volume_id, paper_id=paper_id, width=width
    )


if __name__ == "__main__":
    filename = sys.argv[1]
    outfilename = sys.argv[2]
    tree = etree.parse(filename)
    volume = tree.getroot()
    for paper in volume.findall("paper"):
        if "href" in paper.attrib:
            if not test_url(paper.attrib["href"]):
                sys.stderr.write(
                    "{}:{} removing href attribute: {}\n".format(
                        filename, paper.sourceline, paper.attrib["href"]
                    )
                )
                del paper.attrib["href"]

        href = paper.find("href")
        if href is not None:
            assert len(href) == 0
            if not test_url(href.text):
                sys.stderr.write(
                    "{}:{} removing href element: {}\n".format(
                        filename, href.sourceline, href.text
                    )
                )
                paper.remove(href)

        anth_url = get_anth_url(volume.attrib["id"], int(paper.attrib["id"]))
        anth_url_good = test_url(anth_url)

        url = paper.find("url")
        assert url is None or len(url) == 0

        if url is None:
            if anth_url_good:
                url = etree.Element("url")
                url.text = anth_url
                sys.stderr.write(
                    "{}:{} inserting url element: {}\n".format(
                        filename, paper.sourceline, anth_url
                    )
                )
                paper.append(url)

        else:
            if anth_url_good and url.text != anth_url:
                sys.stderr.write(
                    "{}:{} rewriting url: {} -> {}\n".format(
                        filename, url.sourceline, url.text, anth_url
                    )
                )
                url.text = anth_url

            else:
                sys.stderr.write(
                    "{}:{} removing url element because {} is bad\n".format(
                        filename, url.sourceline, anth_url
                    )
                )
                paper.remove(url)

    tree.write(outfilename, encoding="UTF-8", xml_declaration=True, with_tail=True)


# bin/retract_paper.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Takes an Anthology ID, downloads the PDF, and produces a revision PDF
with a "RETRACTED" watermark, as well as a note at the top pointing
to the paper page. Also revises the XML.
"""

import argparse
import os
import shutil
import subprocess
import sys
import tempfile

from string import Template

from anthology.utils import (
    retrieve_url,
    deconstruct_anthology_id,
    make_simple_element,
    get_xml_file,
    indent,
)
from anthology.data import CANONICAL_URL_TEMPLATE, PDF_LOCATION_TEMPLATE
from add_revision import add_revision

from datetime import datetime

import lxml.etree as ET

template = Template(
    r"""\documentclass{article}
\usepackage[text=RETRACTED,scale=3,color=red]{draftwatermark}
\usepackage{xcolor}
\usepackage{graphicx}
\usepackage{pdfpages}
\usepackage{hyperref}
\hypersetup{plainpages=false,
            pdfpagemode=UseNone,
            colorlinks=true,
            unicode=true
}

% set A4
\setlength{\paperwidth}{21cm}
\setlength{\paperheight}{29.7cm}

\special{papersize=21cm,29.7cm}
\pdfpageheight\paperheight
\pdfpagewidth\paperwidth
\pagestyle{plain}

\begin{document}

\AddToShipoutPicture{%
  \setlength{\unitlength}{1mm}
  % center box at (x, y) millimeters from bottom-left corner
  \put(105,290){\makebox(0,0){This paper was retracted. For more information, see \url{$url}.}}
}

\includepdf[pages=-]{$file}

\end{document}"""
)


def add_watermark(anth_id, workdir="."):
    """
    Downloads an Anthology paper and adds a RETRACTED watermark.
    """
    page = CANONICAL_URL_TEMPLATE.format(anth_id)
    url = PDF_LOCATION_TEMPLATE.format(anth_id)
    orig_pdf = os.path.join(workdir, "tmp.pdf")

    retrieve_url(url, orig_pdf)

    tex_file = os.path.join(workdir, f"{anth_id}.tex")
    print("TEX_FILE", tex_file)
    with open(tex_file, "w") as f:
        print(template.substitute(file=orig_pdf, url=page), file=f)

    command = f"pdflatex {tex_file}"
    try:
        subprocess.call(
            command, shell=True, cwd=workdir, stdout=subprocess.DEVNULL, timeout=30
        )
    except subprocess.TimeoutExpired:
        print(
            "pdflatex didn't finish within 30 seconds. Do you have the CTAN watermark package installed?",
            file=sys.stderr,
        )
        sys.exit(1)

    new_pdf = f"{tex_file}".replace(".tex", ".pdf")

    return new_pdf


def main(args):
    """
    Downloads an Anthology paper and adds a RETRACTED watermark, then updates the XML
    with an appropriate <revision> and <retracted> tag.
    """

    with tempfile.TemporaryDirectory() as tempdir:

        new_pdf = add_watermark(args.anthology_id, workdir=tempdir)

        add_revision(
            args.anthology_id,
            new_pdf,
            explanation="Retracted.",
            change_type="revision",
            dry_run=False,
        )

        xml_file = get_xml_file(args.anthology_id)
        collection_id, volume_id, paper_id = deconstruct_anthology_id(args.anthology_id)
        tree = ET.parse(xml_file)
        if paper_id == "0":
            paper = tree.getroot().find(f"./volume[@id='{volume_id}']/frontmatter")
        else:
            paper = tree.getroot().find(
                f"./volume[@id='{volume_id}']/paper[@id='{paper_id}']"
            )

        if paper is None:
            print(f"Couldn't find paper {args.anthology_id}!", file=sys.stderr)
            sys.exit(2)

        print("Modifying the XML", file=sys.stderr)
        now = datetime.now()
        date = f"{now.year}-{now.month:02d}-{now.day:02d}"
        retracted_node = make_simple_element(
            "retracted", args.explanation, attrib={"date": date}, parent=paper
        )
        indent(tree.getroot())
        tree.write(xml_file, encoding="UTF-8", xml_declaration=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("anthology_id")
    parser.add_argument("explanation", help="Brief description of the changes.")
    args = parser.parse_args()

    main(args)


# bin/revise_revisions.py

#!/usr/bin/env python3

"""
One-time script that converted the old revision style

    <revision id="2">P18-1001v2</revision>

to the new revision style that mandates an explanation

    <revision id="2" href="P18-1001v2">Added new references.</revision>
"""

import lxml.etree as etree
import re
import sys

from anthology.utils import infer_url, test_url, indent

filename = sys.argv[1]
outfilename = sys.argv[2]
tree = etree.parse(filename)
root = tree.getroot()
collection_id = root.attrib["id"]

papers = list(root.findall(".//paper")) + list(root.findall(".//frontmatter"))

for paper in papers:
    for revision in paper.findall("revision"):
        revision.attrib["href"] = revision.text
        revision.text = "No description of the changes were recorded."

indent(root)
tree.write(outfilename, encoding="UTF-8", xml_declaration=True, with_tail=True)


# bin/split_sitemap.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: split_sitemap.py SITEMAP

Split up a sitemap.xml file into multiple files containing at most 50,000 entries each.  (This is Google's upper limit for re-indexing sites via sitemaps.)

Outputs files named SITEMAP.1, SITEMAP.2, etc.

Arguments:
  SITEMAP                  A sitemap file in XML format.
  -h, --help               Display this helpful text.
"""

from copy import deepcopy
from docopt import docopt
from lxml import etree
import os
import sys


SITEMAP_NAMESPACE = "http://www.sitemaps.org/schemas/sitemap/0.9"


if __name__ == "__main__":
    args = docopt(__doc__)

    try:
        sitemap = etree.parse(args["SITEMAP"])
    except Exception as e:
        print("Error parsing sitemap:", file=sys.stderr)
        print(str(e), file=sys.stderr)
        exit(1)

    root = sitemap.getroot()
    chunk_size = 50000
    i = chunk_size
    all_roots, new_root = [], None

    for url in root:
        if i >= chunk_size:
            if new_root is not None:
                all_roots.append(new_root)
            new_root = etree.Element(root.tag, nsmap={None: SITEMAP_NAMESPACE})
            i = 0
        new_root.append(deepcopy(url))
        i += 1

    if new_root is not None:
        all_roots.append(new_root)

    if len(all_roots) > 1:
        print(
            "Split {} entries into {} chunks.".format(
                chunk_size * (len(all_roots) - 1) + i, len(all_roots)
            ),
            file=sys.stderr,
        )

        basename = os.path.splitext(args["SITEMAP"])[0]
        for n, root in enumerate(all_roots):
            with open("{}_{}.xml".format(basename, n + 1), "w") as f:
                print('<?xml version="1.0" encoding="utf-8" standalone="yes" ?>', file=f)
                print(etree.tostring(root, encoding="unicode", pretty_print=True), file=f)
    else:
        print(f"Only found {i} entries, no need to split.", file=sys.stderr)


# bin/summarize_additions.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Takes a git diff on STDIN and pulls out all the attachments and revisions that
were produced, for distribution.

Usage:

  git diff 17fc308a97693a0ce6ec108e398871facb6be78b e9da44eb38de2adfe1bc3cd662ca6e17f124571c | ./bin/summarize_additions.py  | pbcopy

Prints a list to STDOUT.
"""

import argparse
import inflect
import os
import re
import sys

from anthology import Anthology
from anthology.data import ANTHOLOGY_ID_REGEX, CANONICAL_URL_TEMPLATE

from collections import defaultdict


def main(args):
    scriptdir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data')
    anthology = Anthology(importdir=scriptdir)

    attachments = defaultdict(list)
    revisions = []
    errata = []
    for line in sys.stdin:
        if not line.startswith("+"):
            continue

        line = line[1:].strip()
        if line.startswith("<attachment"):
            try:
                match_str = rf'<attachment type="(\w+)">({ANTHOLOGY_ID_REGEX}).*'
                match = re.match(match_str, line)
                attach_type, anthology_id = match.groups()
            except:
                print(f"* Couldn't match '{match_str}' to '{line}'", file=sys.stderr)

            attachments[attach_type].append(
                (
                    anthology.papers[anthology_id].get_title('plain'),
                    CANONICAL_URL_TEMPLATE.format(anthology_id),
                )
            )

        elif line.startswith("<revision"):
            try:
                match_str = rf'<revision.*href="({ANTHOLOGY_ID_REGEX}).*>.*'
                match = re.match(match_str, line)
                anthology_id = match.group(1)
            except:
                print(f"* Couldn't match '{match_str}' to '{line}'", file=sys.stderr)

            paper = anthology.papers[anthology_id]
            explanation = paper.revisions[-1]["explanation"]

            revisions.append(
                (
                    paper.get_title("plain"),
                    CANONICAL_URL_TEMPLATE.format(anthology_id),
                    explanation,
                )
            )

        elif line.startswith("<errat"):
            try:
                match_str = rf"<errat.*?>({ANTHOLOGY_ID_REGEX}).*"
                match = re.match(match_str, line)
                anthology_id = match.group(1)
            except:
                print(f"* Couldn't match '{match_str}' to '{line}'", file=sys.stderr)

            errata.append(
                (
                    anthology.papers[anthology_id].get_title('plain'),
                    CANONICAL_URL_TEMPLATE.format(anthology_id),
                )
            )

    inflector = inflect.engine()
    for attach_type, attachments in attachments.items():
        phrase = inflector.a(attach_type)
        print(f"\nAdded {phrase}:")
        for title, url in attachments:
            print("-", title, "\n ", url, "\n")

    if len(revisions):
        print(f"\nRevisions:")
        for title, url, explanation in revisions:
            print("-", title, "\n ", url, "\n ", explanation, "\n")

    if len(errata):
        print(f"\nErrata:")
        for title, url in errata:
            print("-", title, "\n ", url, "\n")


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    args = parser.parse_args()

    main(args)


# bin/title_case.py

"""author_case.py

usage: author_case.py -o <outdir>

Try to correct titles that are written in mostly uppercase.
"""

import sys
import os.path
import glob
import lxml.etree as etree
import argparse
import logging
import re
from fixedcase.common import get_text


def titlecase(s):
    if s is None:
        return None
    ret = []
    first = True
    for word in re.split(r"([^A-Za-z'’])", s):
        if (
            first
            or word.lower()
            not in "and or nor but a an the as at by for in of on per to vs".split()
        ):
            if len(word) > 0:
                word = word[0].upper() + word[1:].lower()
        else:
            word = word.lower()
        ret.append(word)
        if any(c.isalpha() for c in word):
            first = False
        if word in [':', '(']:
            first = True

    ret = ''.join(ret)
    return ret


def replace_text(node, text):
    def visit(node, skip):
        nonlocal text
        if node.tag == 'fixed-case':
            skip = True
        if node.text:
            n = len(node.text)
            if not skip:
                node.text = text[:n]
            text = text[n:]
        for child in node:
            visit(child, skip)
            if child.tail:
                n = len(child.tail)
                if not skip:
                    child.tail = text[:n]
                text = text[n:]

    visit(node, False)


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO)

    scriptdir = os.path.dirname(os.path.abspath(__file__))
    datadir = os.path.join(scriptdir, '..', 'data')

    ap = argparse.ArgumentParser(description="Correct titles.")
    ap.add_argument(
        "-o", "--outdir", metavar="dir", help="output directory", required=True
    )
    args = ap.parse_args()

    infiles = sorted(glob.glob(os.path.join(datadir, "xml", "*.xml")))

    os.makedirs(os.path.join(args.outdir, "data", "xml"), exist_ok=True)

    for infile in infiles:
        tree = etree.parse(infile)
        root = tree.getroot()
        if not root.tail:
            root.tail = "\n"
        for title in root.xpath(".//title|.//booktitle"):
            title_text = get_text(title)
            ratio = len([c for c in title_text if c == c.lower()]) / len(title_text)
            if ratio < 0.5:
                replace_text(title, titlecase(title_text))
        outfile = os.path.join(args.outdir, "data", "xml", os.path.basename(infile))
        tree.write(outfile, xml_declaration=True, encoding="UTF-8")


# bin/upload_files.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Matt Post <post@cs.jhu.edu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Uploads PDFs and other files to their correct place on aclanthology.org.

Usage:

  [not yet implemented]
  upload_files.py [directory]

will upload all files in the directory to the correct place on the server,
under the assumption they all go to the same folder.

  upload_files.py [file1 [file2 [etc]]]

will upload individual files to their correct place on the server,
where each file may have a different target directory.

"""

import argparse
import os
import re
import subprocess
import sys

from collections import defaultdict
from anthology.utils import deconstruct_anthology_id
from typing import List

# The root directory for files, currently containing pdf/ and attachments/
ANTHOLOGY_FILE_ROOT = "/home/anthologizer/anthology-files"

# The ssh shortcut (in ~/.ssh/config) or full hostname
ANTHOLOGY_HOST = "anth"


def get_dest_path(filepath: str):
    """
    Returns the destination path on the remote server for the file.
    """
    dest_path = ""

    filename = os.path.basename(filepath)
    fileparts = filename.split(".")
    if len(fileparts) == 2:
        # e.g., P19-1001.pdf
        collection_id, volume_id, _ = deconstruct_anthology_id(fileparts[0])
        collection = collection_id[0]
        dest_path = f"pdf/{collection}/{collection_id}"

    elif len(fileparts) == 3:
        # e.g., P19-1001.Attachment.pdf
        collection_id, volume_id, _ = deconstruct_anthology_id(fileparts[0])
        collection = collection_id[0]
        dest_path = f"attachments/{collection}/{collection_id}"

    else:
        raise Exception(f"Can't determine target destination from {filepath}")

    return f"{ANTHOLOGY_HOST}:{ANTHOLOGY_FILE_ROOT}/{dest_path}"


def upload_files(target_uri: str, files: List[str]):
    """
    Uploads regular PDFs or attachments to their correct place on the aclweb server.
    """

    for file in files:
        os.chmod(file, 0o644)

    file_list = " ".join(files)
    command = f"scp -q {file_list} {target_uri}"

    attempts = 1
    retcode = 1
    while attempts <= 3 and retcode != 0:
        # This fails sometimes for no reason, so try a couple of times
        retcode = subprocess.call(command, shell=True)
        if attempts > 1:
            print(f"-> Failed for some reason, attempt #{attempts}", file=sys.stderr)
        print(f"{command} -> {retcode}", file=sys.stderr)
        attempts += 1

    return retcode


def main(args):
    locations = defaultdict(list)
    for file in args.files:
        dest_path = get_dest_path(file)
        locations[dest_path].append(file)

    for location, files in locations.items():
        upload_files(location, files)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("files", nargs="+")
    args = parser.parse_args()

    main(args)


# bin/verify_filelist.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: verify_filelist.py FILELIST [options] [--help]

Compares external files expected by the Anthology to a list of filenames and
their checksums (generated by crc32).

Arguments:
  FILELIST                 Output of `crc32`

Options:
  -r, --list-remaining     Also output files in FILELIST that were not expected.
  --importdir=DIR          Directory to import XML files from. [default: {scriptdir}/../data]
  --debug                  Output debug-level log messages.
  -h, --help               Display this helpful text.
"""

from collections import defaultdict
from docopt import docopt
from glob import glob
from lxml import etree
import logging as log
import os
from urllib.parse import urlparse

from anthology.utils import is_newstyle_id, build_anthology_id, SeverityTracker


def get_expected_path(filetype, filename, collection_id):
    """Maps the relative URL to the expected path on the server.

    THIS IS HIGHLY DEPENDENT ON THE SERVER CONFIGURATION AND .HTACCESS!
    """
    if collection_id[0].isdigit():  # new-style ID
        folder = collection_id.split(".")[-1]
        filename = os.path.join(folder, filename)
    else:
        filename = os.path.join(collection_id[0], collection_id, filename)
    filename = os.path.join(filetype, filename)
    return filename


def is_remote_url(url):
    return bool(urlparse(url).netloc)


def read_checksums(filename):
    checksums = {}
    with open(filename, "r") as f:
        for line in f:
            if not line.strip():
                continue
            crc = line[:8]
            path = line[8:].strip()
            checksums[path] = crc
    return checksums


def main(datadir, crcfile, opts):
    checksums = read_checksums(crcfile)
    problems = defaultdict(int)

    for xml_file in list(glob(f"{datadir}/xml/*.xml")):
        tree = etree.parse(xml_file)
        root = tree.getroot()
        collection_id = root.get("id")

        for element in root.iter("url", "attachment", "revision", "erratum"):
            if element.getparent().tag not in ("paper", "frontmatter", "meta"):
                continue
            if element.tag in ("url", "erratum"):
                path = f"{element.text}.pdf"
                filetype = "pdf"
            elif element.tag == "revision":
                path = f"{element.get('href')}.pdf"
                filetype = "pdf"
            elif element.tag == "attachment":
                path = element.text
                filetype = "attachments"

            if is_remote_url(path):
                continue
            path = get_expected_path(filetype, path, collection_id)
            expected_checksum = element.get("hash")
            if path not in checksums:
                print("\t".join(("missing", filetype, path)))
                problems[filetype] += 1
            elif checksums[path] != expected_checksum:
                print(
                    "\t".join(
                        (
                            "crc-mismatch",
                            filetype,
                            path,
                            expected_checksum,
                            checksums[path],
                        )
                    )
                )
                problems["checksum"] += 1
            else:
                del checksums[path]

    if not problems:
        log.info("No problems detected.")
    if problems["pdf"]:
        log.error(f"{len(problems['pdf']):6d} PDF files missing in list.")
    if problems["attachment"]:
        log.error(f"{len(problems['attachment']):6d} attachments missing in list.")
    if problems["checksum"]:
        log.error(f"{len(problems['checksum']):6d} checksums don't match.")

    if checksums:
        log.warning(f"{len(checksums):6d} files remaining in list that were not used.")
        if opts["list_remaining"]:
            for path in checksums:
                print("\t".join(("unexpected", "", path)))


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )

    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)

    opts = dict(
        list_remaining=bool(args["--list-remaining"]),
    )
    main(args["--importdir"], args["FILELIST"], opts)

    if tracker.highest >= log.ERROR:
        exit(1)


# bin/verify_files.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2020 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: verify_files.py FILE... [options] [--help]

Verify checksum of files downloaded from the Anthology.

Arguments:
  FILE                     File (PDF or attachment) downloaded from the ACL Anthology,
                           with its original filename as stored on the server.

Options:
  --importdir=DIR          Directory to import XML files from. [default: {scriptdir}/../data]
  --debug                  Output debug-level log messages.
  -h, --help               Display this helpful text.
"""

from collections import defaultdict
from docopt import docopt
from glob import glob
from lxml import etree
import logging as log
import os

from anthology.utils import SeverityTracker, compute_hash


def main(datadir, filelist):
    to_check = defaultdict(list)

    for filename in filelist:
        basename = os.path.basename(filename)
        collection_id = basename.split("-")[0]
        with open(filename, "rb") as f:
            checksum = compute_hash(f.read())

        to_check[collection_id].append((basename, checksum))

    for collection_id, checklist in to_check.items():
        xml_file = f"{datadir}/xml/{collection_id}.xml"
        tree = etree.parse(xml_file)
        root = tree.getroot()

        for filename, checksum in checklist:
            if filename.endswith(".pdf"):
                xpath = (
                    f'//attachment[text()="{filename}"] | '
                    f'//url[text()="{filename[:-4]}"] | '
                    f'//erratum[text()="{filename[:-4]}"] | '
                    f'//revision[@href="{filename[:-4]}"]'
                )
            else:
                xpath = f'//attachment[text()="{filename}"]'

            find = etree.XPath(xpath)(root)
            if not find:
                log.error(f"{filename}: couldn't find file in {collection_id}.xml")
                continue
            elif len(find) > 1:
                # this should never happen
                log.warning(
                    f"{filename}: multiple entries with that name in {collection_id}.xml"
                )

            expected = find[0].get("hash")
            if expected != checksum:
                log.error(f"{filename}: CRC32 mismatch -- {checksum} != {expected}")
            else:
                log.debug(f"{filename}: checksum verified")


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )

    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)

    main(args["--importdir"], args["FILE"])

    if tracker.highest >= log.ERROR:
        exit(1)


# bin/vimeo-linker.py

#! /usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2019 Arne Köhn <arne@chark.eu>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: vimeo_linker.py [options]

Searches for videos on the ACL vimeo channel and tries to link them
to papers.

As the vimeo api is not public, you need to copy the
vimeo_apikeys.py.dummy to vimeo_apikeys.py and fill in your API keys
obtained from vimeo.

Arguments:
  --cache-vimeo      Cache the results obtained from vimeo. For debugging.
  --cache-matchings  Cache the paper matchings. For debugging.
  --from-year=N      Papers starting at this year are considered for matching.  Newer year speeds up the matching. [default: 2013]
"""

import difflib
import vimeo
import time
import pickle
import os
import re
from anthology import Anthology
from docopt import docopt

from vimeo_apikeys import *


def checkVideo(paper):
    for elem in paper.attachments:
        if elem["type"] == "video":
            return True
    return False


args = docopt(__doc__)
fromYear = int(args["--from-year"])
cacheVimeo = args["--cache-vimeo"]
cacheMatchings = args["--cache-matchings"]

v = vimeo.VimeoClient(token=personalAccessToken, key=clientId, secret=apiSecret)

allpapers = Anthology(importdir="../data/").papers

print("number of papers in anthology: ", len(allpapers))

papers = {k: v for k, v in allpapers.items() if int(v.attrib["year"]) > fromYear}
print(
    "number of papers in anthology without video after " + str(fromYear) + ": ",
    len(papers),
)


requestUrl = "/users/46432367/videos?per_page=100"
cont = True
nameUrls = {}
numRequests = 0

fetchUrls = True
if cacheVimeo and os.path.isfile("videos.pickle"):
    nameUrls = pickle.load(open("videos.pickle", "rb"))
    fetchUrls = False

while cont and fetchUrls:
    res = v.get(requestUrl)
    if res == None:
        print("Result was None; sleeping and trying again")
        time.sleep(2)
        continue
    numRequests += 1
    j = res.json()
    if not res.ok:
        print("could not fetch videos from vimeo API!")
        exit(1)
    for elem in j["data"]:
        nameUrls[elem["name"]] = elem["link"]
    requestUrl = j["paging"]["next"]
    print(requestUrl)
    cont = requestUrl != None
    # seems to be needed to not run into read timeouts.
    time.sleep(1)

if fetchUrls and cacheVimeo:
    pickle.dump(nameUrls, open("videos.pickle", "wb"))


notFounds = []
result = ""
matcher = difflib.SequenceMatcher()


def trySubstringMatch(videoName):
    for idx, p in papers.items():
        if p.is_volume:
            continue
        title = p.get_title("plain")
        if len(title) <= 20:
            continue
        if title.lower() in videoName:
            return (idx, p)
    return None


def tryMatch(name):
    matcher.set_seq1(name)
    for idx, p in papers.items():
        if p.is_volume:
            continue
        title = p.get_title("plain")
        matcher.set_seq2(title.lower())
        if (
            matcher.real_quick_ratio() > 0.8
            and matcher.quick_ratio() > 0.8
            and matcher.ratio() > 0.8
        ):
            return (idx, p)
    return None


num_elems = len(nameUrls)


id_video = []

computeMatch = True
if os.path.isfile("videos_papers.pickle") and cacheMatchings:
    id_video = pickle.load(open("videos_papers.pickle", "rb"))
    computeMatch = False


if computeMatch:
    i = 1
    for name, url in nameUrls.items():
        print(i, " of ", num_elems, end="")
        i += 1
        # clean tacl tag and author list
        if name.startswith("[TACL]"):
            name = name[6:]
        if "---" in name:
            name = name.split("---")[0]
        found = False
        # skip video names that are obviously not paper titles
        # because they don't contain spaces.
        if " " in name:
            name = name.lower()
            res = trySubstringMatch(name)
            if res == None:
                res = tryMatch(name)
            # try to remove author list appended by : or by -
            if res == None:
                if ":" in name:
                    res = tryMatch(name.rsplit(":", 1)[0])
            if res == None:
                if "-" in name:
                    res = tryMatch(name.rsplit("-", 1)[0])

            if res != None:
                (idx, p) = res
                if checkVideo(p):
                    print("video already exists, skipping ...")
                    continue
                title = p.get_title("plain")
                print("found title " + title + " for video " + name)
                result += (
                    title
                    + "\t"
                    + idx
                    + "\t"
                    + url
                    + "\t"
                    + name
                    + "\t"
                    + p.get_booktitle("plain")
                    + "\n"
                )
                id_video.append((idx, url))
            else:
                notFounds.append((name, url))
                print("no paper found for video" + name)
    f = open("/tmp/vimeo_videos.csv", "w")
    f.write("title\tACL-ID\tvideo url\tvideo name\tpaper title\n")
    f.write(result)
    f.close()
    f = open("/tmp/vimeo_not_matched.csv", "w")
    for name, url in notFounds:
        f.write(url + "\t" + name + "\n")
    f.close()

if computeMatch and cacheMatchings:
    pickle.dump(id_video, open("videos_papers.pickle", "wb"))


venues = set([x[0].split("-")[0] for x in id_video])

id_video_dict = {id: video for id, video in id_video}

paperidre = re.compile(r".*<url>(\w\d\d-\d\d\d\d)</url>")

has_video = False
idx = "notset"
for v in venues:
    with open("../data/xml/" + v + ".xml") as f:
        with open("../data/xml/" + v + ".xml.new", "w") as out:
            for l in f:
                m = paperidre.match(l)
                if m:
                    idx = m[1]  # v+"-"+m[1]
                    has_video = idx in id_video_dict
                if has_video and r"</paper>" in l:
                    out.write(
                        '      <video href="' + id_video_dict[idx] + '" tag="video"/>\n'
                    )
                    has_video = False
                out.write(l)


# bin/volumes_from_diff.py

#!/usr/bin/env python3

"""
Takes a list of XML files on STDIN, and prints all the volumes
within each of those files. e.g.,

    git diff --name-only master | ./bin/volumes_from_xml.py https://preview.aclanthology.org/BRANCH

Used to find the list of volumes to generate previews for.
"""

import sys
import argparse
import lxml.etree as etree
import subprocess


parser = argparse.ArgumentParser()
parser.add_argument("url_root")
args = parser.parse_args()

volumes = []
for filepath in sys.stdin:
    try:
        tree = etree.parse(filepath.rstrip())
    except Exception as e:
        continue
    root = tree.getroot()
    collection_id = root.attrib["id"]
    for volume in root:
        volume_name = volume.attrib["id"]
        volume_id = f"{collection_id}-{volume_name}"
        volumes.append(f"[{volume_id}]({args.url_root}/{volume_id})")

print(", ".join(volumes))


# bin/write_bibkeys_to_xml.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2021 Marcel Bollmann <marcel@bollmann.me>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Usage: write_bibkeys_to_xml.py [--importdir=DIR] [-c | --commit] [--debug]

Generates BibTeX keys for papers that lack them, and writes them to the XML
(if -c|--commit is given).

Options:
  --importdir=DIR          Directory to import XML files from.
                             [default: {scriptdir}/../data/]
  -c, --commit             Commit (=write) the changes to the XML files;
                             will only do a dry run otherwise.
  --debug                  Output debug-level log messages.
  -h, --help               Display this helpful text.
"""

from docopt import docopt
from lxml import etree
import logging as log
import os

from anthology import Anthology
from anthology.utils import SeverityTracker, make_simple_element, indent

import lxml.etree as ET


def write_bibkeys(anthology, srcdir, commit=False):
    for volume_id, volume in anthology.volumes.items():
        papers_without_bibkey = []

        for paper in volume:
            bibkey = paper.bibkey
            if bibkey is None or bibkey == paper.full_id:
                papers_without_bibkey.append(paper)

        if papers_without_bibkey:
            log.info(
                f"Found {len(papers_without_bibkey):4d} papers without bibkeys in volume {volume_id}"
            )
            if not commit:
                continue
        else:
            continue

        # We got some new bibkeys and need to write them to the XML
        xml_file = os.path.join(srcdir, "xml", f"{volume.collection_id}.xml")
        tree = ET.parse(xml_file)
        root = tree.getroot()

        for paper in papers_without_bibkey:
            if paper.paper_id == "0":
                node = root.find(f"./volume[@id='{paper.volume_id}']/frontmatter")
                if node is None:  # dummy frontmatter
                    continue
            else:
                node = root.find(
                    f"./volume[@id='{paper.volume_id}']/paper[@id='{paper.paper_id}']"
                )
            if node is None:
                log.error(f"Paper {paper.full_id} not found in {xml_file}")
                continue

            # Generate unique bibkey
            bibkey = anthology.pindex.create_bibkey(paper, vidx=anthology.venues)
            make_simple_element("bibkey", bibkey, parent=node)

        indent(root)
        tree.write(xml_file, encoding="UTF-8", xml_declaration=True)


if __name__ == "__main__":
    args = docopt(__doc__)
    scriptdir = os.path.dirname(os.path.abspath(__file__))
    if "{scriptdir}" in args["--importdir"]:
        args["--importdir"] = os.path.abspath(
            args["--importdir"].format(scriptdir=scriptdir)
        )

    log_level = log.DEBUG if args["--debug"] else log.INFO
    log.basicConfig(format="%(levelname)-8s %(message)s", level=log_level)
    tracker = SeverityTracker()
    log.getLogger().addHandler(tracker)

    log.info("Instantiating the Anthology...")
    anthology = Anthology(importdir=args["--importdir"], require_bibkeys=False)
    log.info("Scanning for papers without <bibkey> tags...")
    write_bibkeys(anthology, args["--importdir"], commit=bool(args["--commit"]))

    if not args["--commit"]:
        if tracker.highest >= log.ERROR:
            log.warning(
                "There were errors! Please check them carefully before re-running this script with -c/--commit."
            )
        else:
            log.warning(
                "Re-run this script with -c/--commit to save these changes to the XML files."
            )

    if tracker.highest >= log.ERROR:
        exit(1)


# bin/yamlfix.py

# Workaround for bug in PyYAML

import yaml
import re

for name in ["Dumper", "SafeDumper", "CDumper", "CSafeDumper"]:
    try:
        Dumper = yaml.__dict__[name]
    except KeyError:
        continue
    Dumper.add_implicit_resolver("tag:yaml.org,2002:bool", re.compile("^[YNyn]$"), "YNyn")


# bin/ingest_aclpub2.py
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2021, 2022 Xinru Yan <xinru1414@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Usage:
#   python bin/ingest_aclpub2.py
#
#
#
from collections import defaultdict
import click
import yaml
import re
import sys
import os
import glob
import PyPDF2
from pathlib import Path
from datetime import datetime
from collections import defaultdict
from itertools import chain
import lxml.etree as etree
from typing import Dict, List, Tuple, Any, Optional
from ingest import maybe_copy

from normalize_anth import normalize
from anthology.index import AnthologyIndex
from anthology.venues import VenueIndex
from anthology.people import PersonName
from anthology.utils import (
    make_simple_element,
    indent,
    compute_hash_from_file,
)


def disambiguate_name(node, anth_id, people):
    name = PersonName.from_element(node)
    ids = people.get_ids(name)
    choice = -1
    if len(ids) > 1:
        while choice < 0 or choice >= len(ids):
            print(
                f'({anth_id}): ambiguous author {name}; Please choose from the following:'
            )
            for i, id_ in enumerate(ids):
                print(f'[{i}] {id_} ({people.get_comment(id_)})')
            choice = int(input("--> "))

    return ids[choice], choice


def correct_caps(name):
    '''
    Many people submit their names in "ALL CAPS" or "all lowercase".
    Correct this with heuristics.
    '''
    if name.islower() or name.isupper():
        # capitalize all parts
        corrected = " ".join(list(map(lambda x: x.capitalize(), name.split())))
        print(
            f"-> Correcting capitalization of '{name}' to '{corrected}'",
            file=sys.stderr,
        )
        name = corrected
    return name


def parse_conf_yaml(ingestion_dir: str) -> Dict[str, Any]:
    '''
    poss meta keys = [
                 'book_title',
                 'event_name',
                 'cover_subtitle',
                 'anthology_venue_id',
                 'volume',
                 'start_date',
                 'end_date',
                 'isbn',
                 'location',
                 'editors',
                 'publisher'
                 ]
    must meta keys = [
                 'book_title',
                 'anthology_venue_id',
                 'volume',
                 'month',
                 'year',
                 'location',
                 'editors',
                 'publisher'
                ]

    anthology_venue_id == abbrev
    event_name == title
    cover_subtitle == shortbooktitle
    '''
    if os.path.exists(Path(ingestion_dir + 'inputs/conference_details.yml')):
        meta = yaml.safe_load(
            Path(ingestion_dir + 'inputs/conference_details.yml').read_text()
        )
    else:
        meta = yaml.safe_load(
            Path(ingestion_dir + 'input/conference_details.yml').read_text()
        )
    meta['month'] = meta['start_date'].strftime('%B')
    meta['year'] = str(meta['start_date'].year)

    must_have_keys = [
        'book_title',
        'anthology_venue_id',
        'volume',
        'month',
        'year',
        'location',
        'editors',
        'publisher',
        'event_name',
    ]
    for key in must_have_keys:
        assert key in meta.keys(), f'{key} is missing in the conference_details.yml file'

    if re.match(rf'^[a-z0-9]+$', meta['volume']) is None:
        raise Exception(
            f"Invalid volume key '{meta['volume']}' in {ingestion_dir + 'inputs/conference_details.yml'}"
        )

    return meta


def parse_paper_yaml(ingestion_dir: str) -> List[Dict[str, str]]:
    if os.path.exists(Path(ingestion_dir + 'inputs/conference_details.yml')):
        papers = yaml.safe_load(Path(ingestion_dir + 'inputs/papers.yml').read_text())
    else:
        papers = yaml.safe_load(Path(ingestion_dir + 'input/papers.yml').read_text())
    return papers


def add_paper_nums_in_paper_yaml(
    papers: List[Dict[str, str]], ingestion_dir: str
) -> List[Dict[str, str]]:
    start, end = 1, 0
    for paper in papers:
        paper_id = str(paper['id'])
        paper_name = paper['file']
        if os.path.exists(f'{ingestion_dir}inputs/papers/{paper_id}.pdf'):
            paper_need_read_path = f'{ingestion_dir}inputs/papers/{paper_id}.pdf'
        elif os.path.exists(f'{ingestion_dir}input/papers/{paper_id}.pdf'):
            paper_need_read_path = f'{ingestion_dir}input/papers/{paper_id}.pdf'
        elif os.path.exists(f'{ingestion_dir}inputs/papers/{paper_name}'):
            paper_need_read_path = f'{ingestion_dir}inputs/papers/{paper_name}'
        elif os.path.exists(f'{ingestion_dir}input/papers/{paper_name}'):
            paper_need_read_path = f'{ingestion_dir}input/papers/{paper_name}'
        else:
            paper_need_read_path = None
        assert paper_need_read_path, f'{paper_id} path is None'
        pdf = open(paper_need_read_path, 'rb')
        pdf_reader = PyPDF2.PdfFileReader(pdf)
        start = end + 1
        end = start + pdf_reader.numPages - 1
        paper['pages'] = f'{start}-{end}'
    return papers


def create_des_path(org_dir_name: str, venue_name: str) -> str:
    dest_dir = os.path.join(org_dir_name, venue_name)
    if not os.path.exists(dest_dir):
        os.makedirs(dest_dir)
    return dest_dir


def find_paper_attachment(paper_name: str, attachments_dir: str) -> Optional[str]:
    '''
    files in the attachments folder need to be named filename.zip
    '''
    attachment_path = None
    for filename in glob.glob(attachments_dir + '/*'):
        if os.path.splitext(os.path.split(filename)[1])[0] == paper_name:
            attachment_path = filename
            break
    return attachment_path


def proceeding2xml(anthology_id: str, meta: Dict[str, Any]):
    fields = [
        'editor',
        'booktitle',
        'month',
        'year',
        'url',
    ]
    paper = make_simple_element('paper', attrib={'id': '0'})
    for field in fields:
        if field == 'editor':
            authors = meta['editors']
            for author in authors:
                name_node = make_simple_element(field, parent=paper)
                make_simple_element('first', author['first_name'], parent=name_node)
                make_simple_element('last', author['last_name'], parent=name_node)
        else:
            if field == 'url':
                value = f'{anthology_id}'
            elif field == 'booktitle':
                value = meta['book_title']
            elif field == 'month':
                value = meta['month']
            elif field == 'year':
                value = meta['year']

            try:
                make_simple_element(field, text=value, parent=paper)
            except:
                print(f"Couldn't process {paper} for {anthology_id}", file=sys.stderr)
                sys.exit(2)
    return paper


def paper2xml(
    paper_item: Dict[str, str], paper_num: int, anthology_id: str, meta: Dict[str, Any]
):
    '''
    paper keys = ['abstract',
                  'attachments',
                  'attributes',
                  'authors',
                  'decision',
                  'file',
                  'id',
                  'openreview_id',
                  'pdf_file',
                  'title']
    author keys = ['emails',
                   'first_name',
                   'google_scholar_id',
                   'homepage',
                   'last_name',
                   'name',
                   'semantic_scholar_id',
                   'username']
    '''
    fields = [
        'title',
        'author',
        'pages',
        'abstract',
        'url',
        'doi',
        'language',
    ]
    paper = make_simple_element('paper', attrib={'id': str(paper_num)})
    for field in fields:
        if field == 'author':
            authors = paper_item['authors']
            for author in authors:
                name_node = make_simple_element(field, parent=paper)
                make_simple_element('first', author['first_name'], parent=name_node)
                make_simple_element('last', author['last_name'], parent=name_node)
        else:
            if field == 'url':
                value = f'{anthology_id}'
            elif field == 'abstract':
                value = paper_item['abstract'].replace('\n', '')
            elif field == 'title':
                value = paper_item[field]
            elif field == 'pages':
                value = paper_item[field]
            else:
                continue

            try:
                make_simple_element(field, text=value, parent=paper)
            except:
                print(f"Couldn't process {paper} for {anthology_id}", file=sys.stderr)
                sys.exit(2)
    return paper


def process_procedding(
    ingestion_dir: str,
    anthology_datadir: str,
    venue_index: VenueIndex,
    venue_keys: List[str],
) -> Tuple[str, Dict[str, Any]]:
    meta = parse_conf_yaml(ingestion_dir)
    venue_abbrev = meta["anthology_venue_id"]
    venue_slug = venue_index.get_slug(venue_abbrev)

    if str(datetime.now().year) in venue_abbrev:
        print(f"Fatal: Venue assembler put year in acronym: '{venue_abbrev}'")
        sys.exit(1)

    if re.match(r".*\d$", venue_abbrev) is not None:
        print(
            f"WARNING: Venue {venue_abbrev} ends in a number, this is probably a mistake"
        )

    if venue_slug not in venue_keys:
        event_name = meta['event_name']
        assert (
            re.match(r'(.)* [Ww]orkshop', event_name) is None
        ), f"event name should start with Workshop or The Workshop, instead it started with {re.match(r'(.)* [Ww]orkshop', event_name)[0]}"
        venue_index.add_venue(venue_abbrev, meta['event_name'])
        print(f"Creating venue '{venue_abbrev}' ({event_name})")
        venue_index.dump(directory=anthology_datadir)

    meta["path"] = ingestion_dir
    meta["collection_id"] = collection_id = meta["year"] + "." + venue_slug
    volume_name = meta["volume"].lower()
    volume_full_id = f"{collection_id}-{volume_name}"

    # if "sig" in meta:
    #     print(
    #         f"Add this line to {anthology_datadir}/sigs/{meta['sig'].lower()}.yaml:"
    #     )
    #     print(f"  - {meta['year']}:")
    #     print(f"    - {volume_full_id} # {meta['booktitle']}")

    # print(f'volume_full_id {volume_full_id} meta {meta}')
    return volume_full_id, meta


def copy_pdf_and_attachment(
    meta: Dict[str, Any],
    pdfs_dir: str,
    attachments_dir: str,
    papers: List[Dict[str, str]],
    dry_run: bool,
) -> Tuple[Dict[str, Dict[str, str]], str, str, str]:
    volume = dict()
    collection_id = meta['collection_id']
    venue_name = meta['anthology_venue_id'].lower()
    volume_name = meta['volume'].lower()

    pdfs_dest_dir = create_des_path(pdfs_dir, venue_name)

    pdfs_src_dir = os.path.join(meta['path'], 'watermarked_pdfs')

    # copy proceedings.pdf
    proceedings_pdf_src_path = os.path.join(meta['path'], 'proceedings.pdf')
    assert os.path.exists(proceedings_pdf_src_path), f'proceedings.pdf was not found'
    proceedings_pdf_dest_path = (
        os.path.join(pdfs_dest_dir, f"{collection_id}-{volume_name}") + ".pdf"
    )
    if dry_run:
        print(
            f'would\'ve moved {proceedings_pdf_src_path} to {proceedings_pdf_dest_path}'
        )
    if not dry_run:
        maybe_copy(proceedings_pdf_src_path, proceedings_pdf_dest_path)

    paper_id_full = f'{collection_id}-{volume_name}.0'
    volume[0] = {
        'anthology_id': paper_id_full,
        'pdf': proceedings_pdf_dest_path,
        'attachments': [],
    }

    for i, paper in enumerate(papers):
        # copy pdf
        paper_name = paper['file']
        paper_id = str(paper['id'])
        paper_num = i + 1
        paper_id_full = f'{collection_id}-{volume_name}.{paper_num}'

        if os.path.exists(os.path.join(pdfs_src_dir, paper_name)):
            pdf_src_path = os.path.join(pdfs_src_dir, paper_name)
        elif os.path.exists(os.path.join(pdfs_src_dir, f'{paper_id}.pdf')):
            pdf_src_path = os.path.join(pdfs_src_dir, f'{paper_id}.pdf')
        else:
            pdf_src_path = None
        assert pdf_src_path, f'{paper_name} path is None'
        pdf_dest_path = os.path.join(
            pdfs_dest_dir, f"{collection_id}-{volume_name}.{paper_num}.pdf"
        )
        if dry_run:
            print(f'would\'ve moved {pdf_src_path} to {pdf_dest_path}')
        if not dry_run:

            maybe_copy(pdf_src_path, pdf_dest_path)

        volume[paper_num] = {
            'anthology_id': paper_id_full,
            'pdf': pdf_dest_path,
            'attachments': [],
        }
        # copy attachments
        if 'attachments' in paper.keys():
            attchs_dest_dir = create_des_path(attachments_dir, venue_name)
            attchs_src_dir = os.path.join(meta['path'], 'attachments')
            assert os.path.exists(
                attchs_src_dir
            ), f'paper {i, paper_name} contains attachments but attachments folder was not found'
            attch_src_path = find_paper_attachment(
                str(os.path.splitext(paper_name)[0]), attchs_src_dir
            )
            print(f'{attch_src_path}')
            assert attch_src_path, f'{paper_name} attachment path is None'
            _, attch_src_extension = os.path.splitext(attch_src_path)
            type_ = paper['attachments'][0]['type']
            file_name = (
                f'{collection_id}-{volume_name}.{paper_num}.{type_}{attch_src_extension}'
            )
            attch_dest_path = os.path.join(attchs_dest_dir, file_name)

            if dry_run:
                print(f'would\'ve moved {attch_src_path} to {attch_dest_path}')
            if not dry_run:
                maybe_copy(attch_src_path, attch_dest_path)
            volume[paper_num]['attachments'].append((attch_dest_path, type_))
    return volume, collection_id, volume_name, proceedings_pdf_dest_path


def create_xml(
    volume: Dict[str, Dict[str, str]],
    anthology_dir: str,
    ingest_date: str,
    collection_id: str,
    volume_name: str,
    meta: Dict[str, Any],
    proceedings_pdf_dest_path: str,
    people,
    papers: List[Dict[str, str]],
) -> None:
    collection_file = os.path.join(anthology_dir, 'data', 'xml', f'{collection_id}.xml')
    if os.path.exists(collection_file):
        root_node = etree.parse(collection_file).getroot()
    else:
        root_node = make_simple_element('collection', attrib={'id': collection_id})

    volume_node = make_simple_element(
        'volume',
        attrib={'id': volume_name, 'ingest-date': ingest_date},
    )
    # Replace the existing one if present
    existing_volume_node = root_node.find(f"./volume[@id='{volume_name}']")
    for i, child in enumerate(root_node):
        if child.attrib['id'] == volume_name:
            root_node[i] = volume_node
            break
    else:
        root_node.append(volume_node)

    meta_node = None

    for paper_num, paper in sorted(volume.items()):
        paper_id_full = paper['anthology_id']
        # print(f'creating xml for paper name {paper}, in papers {papers[paper_num-1]}')
        if paper_num == 0:
            paper_node = proceeding2xml(paper_id_full, meta)
        else:
            paper_node = paper2xml(papers[paper_num - 1], paper_num, paper_id_full, meta)

        if paper_node.attrib['id'] == '0':
            # create metadata subtree
            meta_node = make_simple_element('meta', parent=volume_node)
            title_node = paper_node.find('booktitle')
            meta_node.append(title_node)
            for editor in paper_node.findall('./editor'):
                disamb_name, name_choice = disambiguate_name(
                    editor, paper_id_full, people
                )
                if name_choice != -1:
                    editor.attrib['id'] = disamb_name
                person = PersonName.from_element(editor)
                for name_part in editor:
                    name_part.text = correct_caps(name_part.text)
                meta_node.append(editor)

            # Get the publisher from the meta file
            publisher_node = make_simple_element('publisher', meta['publisher'])
            meta_node.append(publisher_node)

            # Get the address from the meta file
            address_node = make_simple_element("address", meta['location'])
            meta_node.append(address_node)

            meta_node.append(paper_node.find('month'))
            meta_node.append(paper_node.find('year'))

            make_simple_element(
                'url',
                text=f"{collection_id}-{volume_name}",
                attrib={'hash': compute_hash_from_file(proceedings_pdf_dest_path)},
                parent=meta_node,
            )

            # modify frontmatter tag
            paper_node.tag = 'frontmatter'
            del paper_node.attrib['id']

        url = paper_node.find('./url')
        # if url is not None:
        url.attrib['hash'] = compute_hash_from_file(paper['pdf'])

        for path, type_ in paper['attachments']:
            make_simple_element(
                'attachment',
                text=os.path.basename(path),
                attrib={
                    'type': type_,
                    'hash': compute_hash_from_file(path),
                },
                parent=paper_node,
            )

        if len(paper_node) > 0:
            volume_node.append(paper_node)

        # Normalize
        for oldnode in paper_node:
            normalize(oldnode, informat='latex')

        # Adjust the language tag
        # language_node = paper_node.find('./language')
        # if language_node is not None:
        #     try:
        #         lang = iso639.languages.get(name=language_node.text)
        #     except KeyError:
        #         raise Exception(f"Can't find language '{language_node.text}'")
        #     language_node.text = lang.part3

        # Fix author names
        for name_node in paper_node.findall('./author'):
            disamb_name, name_choice = disambiguate_name(name_node, paper_id_full, people)
            if name_choice != -1:
                name_node.attrib['id'] = disamb_name
            person = PersonName.from_element(name_node)
            for name_part in name_node:
                name_part.text = correct_caps(name_part.text)

    # Other data from the meta file
    # if 'isbn' in meta:
    #     make_simple_element('isbn', meta['isbn'], parent=meta_node)

    indent(root_node)
    tree = etree.ElementTree(root_node)
    tree.write(collection_file, encoding='UTF-8', xml_declaration=True, with_tail=True)


@click.command()
@click.option(
    '-i',
    '--ingestion_dir',
    help='Directory contains proceedings need to be ingested',
)
@click.option(
    '-p',
    '--pdfs_dir',
    default=os.path.join(os.environ['HOME'], 'anthology-files', 'pdf'),
    help='Root path for placement of PDF files',
)
@click.option(
    '-a',
    '--attachments_dir',
    default=os.path.join(os.environ['HOME'], 'anthology-files', 'attachments'),
    help='Root path for placement of attachment files',
)
@click.option(
    '-n',
    '--dry_run',
    default=False,
    help='Do not actually copy anything',
)
@click.option(
    '-r',
    '--anthology_dir',
    default=os.path.join(os.path.dirname(sys.argv[0]), ".."),
    help='Root path of ACL Anthology Github repo.',
)
@click.option(
    '-d',
    '--ingest_date',
    default=f'{datetime.now().year}-{datetime.now().month:02d}-{datetime.now().day:02d}',
    help='Ingestion date',
)
def main(ingestion_dir, pdfs_dir, attachments_dir, dry_run, anthology_dir, ingest_date):
    anthology_datadir = os.path.join(os.path.dirname(sys.argv[0]), "..", "data")
    venue_index = VenueIndex(srcdir=anthology_datadir)
    venue_keys = [venue["slug"].lower() for _, venue in venue_index.items()]

    people = AnthologyIndex(srcdir=anthology_datadir)
    # people.bibkeys = load_bibkeys(anthology_datadir)

    volume_full_id, meta = process_procedding(
        ingestion_dir, anthology_datadir, venue_index, venue_keys
    )
    papers = parse_paper_yaml(ingestion_dir)
    # print(f'original paper {papers[0]}')
    papers = add_paper_nums_in_paper_yaml(papers, ingestion_dir)
    # print(f'updated paper {papers[0]}')
    (
        volume,
        collection_id,
        volume_name,
        proceedings_pdf_dest_path,
    ) = copy_pdf_and_attachment(meta, pdfs_dir, attachments_dir, papers, dry_run)
    create_xml(
        volume=volume,
        anthology_dir=anthology_dir,
        ingest_date=ingest_date,
        collection_id=collection_id,
        volume_name=volume_name,
        meta=meta,
        proceedings_pdf_dest_path=proceedings_pdf_dest_path,
        people=people,
        papers=papers,
    )


if __name__ == '__main__':
    main()
