Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions spacy/lang/cy/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from ...language import BaseDefaults, Language
from .lex_attrs import LEX_ATTRS
from .punctuation import TOKENIZER_INFIXES, TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES
from .stop_words import STOP_WORDS
from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS


class WelshDefaults(BaseDefaults):
tokenizer_exceptions = TOKENIZER_EXCEPTIONS
prefixes = TOKENIZER_PREFIXES
suffixes = TOKENIZER_SUFFIXES
infixes = TOKENIZER_INFIXES
lex_attr_getters = LEX_ATTRS
stop_words = STOP_WORDS


class Welsh(Language):
lang = "cy"
Defaults = WelshDefaults


__all__ = ["Welsh"]
16 changes: 16 additions & 0 deletions spacy/lang/cy/examples.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""
Example sentences to test spaCy and its language models.

>>> from spacy.lang.cy.examples import sentences
>>> docs = nlp.pipe(sentences)
"""

sentences = [
"Caerdydd yw prifddinas Cymru.",
"Aeth hi i'r siop i brynu bara a llaeth.",
"Mae'r cyd-destun yn bwysig wrth gyfieithu.",
"Ganwyd Dylan Thomas yn Abertawe yn 1914.",
"Ble mae'r orsaf drenau agosaf?",
"Faint o'r gloch yw hi?",
"Beth yw'r tywydd heddiw?",
]
48 changes: 48 additions & 0 deletions spacy/lang/cy/lex_attrs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from typing import Any, Callable, Dict

from ...attrs import LIKE_NUM

# Number words with their soft, nasal and aspirate mutations.
# fmt: off
_num_words = {
"sero",
"un",
"dau", "ddau", "nau", "dwy", "ddwy", "nwy",
"tri", "dri", "thri", "tair", "dair", "thair",
"pedwar", "bedwar", "phedwar", "pedair", "bedair", "phedair",
"pump", "bump", "phump", "pum", "bum", "phum",
"chwech", "chwe",
"saith",
"wyth",
"naw",
"deg", "ddeg", "deng", "ddeng",
"deuddeg", "ddeuddeg", "deuddeng", "ddeuddeng",
"pymtheg", "bymtheg", "phymtheg", "pymtheng", "bymtheng", "phymtheng",
"deunaw", "ddeunaw", "neunaw",
"ugain", "hugain",
"deugain", "ddeugain", "neugain",
"trigain", "drigain", "thrigain",
"cant", "gant", "chant", "can", "gan", "chan",
"mil", "fil",
"miliwn", "filiwn",
"biliwn",
"triliwn", "driliwn", "thriliwn",
"cwadriliwn", "gwadriliwn", "chwadriliwn",
}
# fmt: on


def like_num(text: str) -> bool:
if text.startswith(("+", "-", "±", "~")):
text = text[1:]
text = text.replace(",", "").replace(".", "")
if text.isdigit():
return True
if text.count("/") == 1:
num, denom = text.split("/")
if num.isdigit() and denom.isdigit():
return True
return text.lower() in _num_words


LEX_ATTRS: Dict[int, Callable[[str], Any]] = {LIKE_NUM: like_num}
45 changes: 45 additions & 0 deletions spacy/lang/cy/punctuation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
from ..char_classes import (
ALPHA,
ALPHA_LOWER,
ALPHA_UPPER,
CONCAT_QUOTES,
LIST_ELLIPSES,
LIST_ICONS,
)
from ..punctuation import (
TOKENIZER_PREFIXES as BASE_PREFIXES,
TOKENIZER_SUFFIXES as BASE_SUFFIXES,
)

# Welsh clitics are written with an apostrophe (straight or curly) and attach
# to the preceding word ('r, 'n, 'i, 'm, 'ch, 'th, 'u, 'w) or, for f' and d',
# to the following word. They are tokenized as separate tokens, as in the UD
# Welsh treebank.
_APOSTROPHES = "'’"

_prefixes = [r"[fFdD][{ap}]".format(ap=_APOSTROPHES)]

_suffixes = [
r"[{ap}](?:ch|th|CH|TH|[rnimuwRNIMUW])".format(ap=_APOSTROPHES),
]

# The base infixes split on a hyphen between letters. Welsh hyphenates within
# words to mark stress and to separate digraphs from letter pairs (cyd-destun
# has two Ds, cuddio one DD), so that rule is left out; the hyphen between
# digits is kept.
_infixes = (
LIST_ELLIPSES
+ LIST_ICONS
+ [
r"(?<=[0-9])[+\-\*^](?=[0-9-])",
r"(?<=[{al}{q}])\.(?=[{au}{q}])".format(
al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES
),
r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA),
r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA),
]
)

TOKENIZER_PREFIXES = BASE_PREFIXES + _prefixes
TOKENIZER_SUFFIXES = BASE_SUFFIXES + _suffixes
TOKENIZER_INFIXES = _infixes
104 changes: 104 additions & 0 deletions spacy/lang/cy/stop_words.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
ADPS = {
"'n",
"ag",
"am",
"ar",
"arnaf",
"at",
"dan",
"dros",
"drwy",
"gan",
"gyda",
"heb",
"hyd",
"i",
"mewn",
"o",
"tros",
"wedi",
"wrth",
"ym",
"yn",
"yng",
"â",
}

ADVS = {
"beth",
"ble",
"llawer",
"lle",
"yma",
"yna",
}

AUXS = {
"bod",
"byddaf",
"mae",
"sy",
"sydd",
"ydwyf",
"yw",
}

# Also content words: "maen" (a stone), "oes" (an age).
AMBIGS = {
"maen",
"oes",
}

CONJS = {
"a",
"ac",
"fel",
"neu",
"ond",
"os",
"tra",
}

DETS = {
"'r",
"un",
"y",
"yr",
}

NOUNS = {
"ôl",
"tu",
}

PARTS = {
"na",
"nid",
"nis",
}

PRONS = {
"di",
"dy",
"ef",
"ei",
"eich",
"ein",
"eu",
"fe",
"fy",
"hi",
"hun",
"hwy",
"hyn",
"i",
"nhw",
"ni",
"pob",
}

STOP_WORDS = set(ADPS | ADVS | AUXS | AMBIGS | CONJS | DETS | NOUNS | PARTS | PRONS)

# The tokenizer splits clitics written with a curly apostrophe like the
# straight forms, so both spellings are stop words.
STOP_WORDS.update({word.replace("'", "’") for word in STOP_WORDS if "'" in word})
25 changes: 25 additions & 0 deletions spacy/lang/cy/tokenizer_exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
from ...symbols import NORM, ORTH
from ...util import update_exc
from ..tokenizer_exceptions import BASE_EXCEPTIONS

_exc = {}

for exc_data in [
{ORTH: "e.e.", NORM: "er enghraifft"},
{ORTH: "h.y.", NORM: "hynny yw"},
{ORTH: "a.y.b.", NORM: "ac yn y blaen"},
{ORTH: "Dr.", NORM: "Doctor"},
{ORTH: "Mr.", NORM: "Mister"},
{ORTH: "Mrs.", NORM: "Missus"},
]:
_exc[exc_data[ORTH]] = [exc_data]

# A clitic standing on its own (after a space, or once split off by the
# suffix rules) must not lose its apostrophe to the quote prefix rule.
for clitic in ["r", "n", "i", "m", "ch", "th", "u", "w"]:
for apostrophe in ["'", "’"]:
for form in [clitic, clitic.upper()]:
orth = apostrophe + form
_exc[orth] = [{ORTH: orth}]

TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)
5 changes: 5 additions & 0 deletions spacy/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ def cs_tokenizer():
return get_lang_class("cs")().tokenizer


@pytest.fixture(scope="session")
def cy_tokenizer():
return get_lang_class("cy")().tokenizer


@pytest.fixture(scope="session")
def da_tokenizer():
return get_lang_class("da")().tokenizer
Expand Down
Empty file added spacy/tests/lang/cy/__init__.py
Empty file.
27 changes: 27 additions & 0 deletions spacy/tests/lang/cy/test_exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import pytest


@pytest.mark.parametrize(
"text,norm",
[
("e.e.", "er enghraifft"),
("h.y.", "hynny yw"),
("a.y.b.", "ac yn y blaen"),
],
)
def test_cy_tokenizer_handles_abbreviation(cy_tokenizer, text, norm):
tokens = cy_tokenizer(text)
assert len(tokens) == 1
assert tokens[0].text == text
assert tokens[0].norm_ == norm


def test_cy_tokenizer_abbreviation_in_sentence(cy_tokenizer):
tokens = [t.text for t in cy_tokenizer("Ffrwythau, e.e. afalau a gellyg.")]
assert tokens == ["Ffrwythau", ",", "e.e.", "afalau", "a", "gellyg", "."]


@pytest.mark.parametrize("text", ["Dr.", "Mr.", "Mrs."])
def test_cy_tokenizer_handles_title_abbreviation(cy_tokenizer, text):
tokens = cy_tokenizer(text + " Jones")
assert [t.text for t in tokens] == [text, "Jones"]
48 changes: 48 additions & 0 deletions spacy/tests/lang/cy/test_lex_attrs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import pytest

from spacy.lang.cy.lex_attrs import like_num


@pytest.mark.parametrize("text", ["10", "1,000", "3.5", "1/2", "-7", "+12"])
def test_cy_like_num_digits(text):
assert like_num(text)


@pytest.mark.parametrize(
"text",
[
"un",
"dau",
"ddwy",
"tri",
"thair",
"pedwar",
"bum",
"chwech",
"deg",
"ddeg",
"deuddeg",
"pymtheg",
"deunaw",
"ugain",
"hugain",
"cant",
"gant",
"mil",
"miliwn",
"Deg",
"UGAIN",
],
)
def test_cy_like_num_number_words(text):
assert like_num(text)


@pytest.mark.parametrize("text", ["ar", "cath", "a", "deng-", "unwaith"])
def test_cy_like_num_not_number(text):
assert not like_num(text)


def test_cy_tokenizer_marks_number_words(cy_tokenizer):
tokens = cy_tokenizer("un ar ddeg o gathod")
assert [t.like_num for t in tokens] == [True, False, True, False, False]
Loading