sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2024-12-25 20:04:17 +00:00
Code Issues Projects Releases Wiki Activity

Use smaller models

Browse Source
This commit is contained in:
Patrick Jentsch 2020-09-23 15:46:43 +02:00
parent 1ed42f68ad
commit 8afdfb13b2
4 changed files with 29 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitlab-ci.yml
View File

@ -3,16 +3,12 @@ image: docker:stable
services:
- docker:stable-dind
variables:
DOCKER_DRIVER: overlay2
stages:
- build
- push
.docker_setup:
before_script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
variables:
DOCKER_DRIVER: overlay2
.reg_setup:
before_script:
@ -25,8 +21,8 @@ stages:
REG_VERSION: 0.16.1
build_image:
extends: .docker_setup
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $INTERMEDIATE_IMAGE_TAG .
- docker push $INTERMEDIATE_IMAGE_TAG
stage: build
@ -37,11 +33,11 @@ build_image:
push_master:
extends:
- .docker_setup
- .reg_setup
only:
- master
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $INTERMEDIATE_IMAGE_TAG
- /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $INTERMEDIATE_IMAGE_TAG
- docker tag $INTERMEDIATE_IMAGE_TAG $IMAGE_TAG
@ -55,7 +51,6 @@ push_master:
push_other:
extends:
- .docker_setup
- .reg_setup
except:
- master
@ -63,6 +58,7 @@ push_other:
- branches
- tags
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $INTERMEDIATE_IMAGE_TAG
- /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $INTERMEDIATE_IMAGE_TAG
- docker tag $INTERMEDIATE_IMAGE_TAG $IMAGE_TAG

16
Dockerfile
View File

@ -34,14 +34,14 @@ RUN apt-get install -y --no-install-recommends \
setuptools \
wheel \
&& pip3 install "spacy==${SPACY_VERSION}" \
&& python3 -m spacy download "de_core_news_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "el_core_news_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "en_core_web_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "es_core_news_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "fr_core_news_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "it_core_news_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "nl_core_news_lg-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "pt_core_news_lg-${SPACY_MODELS_VERSION}" --direct
&& python3 -m spacy download "de_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "el_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "en_core_web_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "es_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "fr_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "it_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "nl_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "pt_core_news_md-${SPACY_MODELS_VERSION}" --direct
COPY nlp /usr/local/bin
COPY spacy-nlp /usr/local/bin

16
nlp
View File

@ -16,14 +16,14 @@ import os
import sys
SPACY_MODELS = {'de': 'de_core_news_sm',
'el': 'el_core_news_sm',
'en': 'en_core_web_sm',
'es': 'es_core_news_sm',
'fr': 'fr_core_news_sm',
'it': 'it_core_news_sm',
'nl': 'nl_core_news_sm',
'pt': 'pt_core_news_sm'}
SPACY_MODELS = {'de': 'de_core_news_md',
'el': 'el_core_news_md',
'en': 'en_core_web_md',
'es': 'es_core_news_md',
'fr': 'fr_core_news_md',
'it': 'it_core_news_md',
'nl': 'nl_core_news_md',
'pt': 'pt_core_news_md'}
def parse_args():

16
spacy-nlp
View File

@ -9,14 +9,14 @@ import os
import spacy
import textwrap
SPACY_MODELS = {'de': 'de_core_news_lg',
'el': 'el_core_news_lg',
'en': 'en_core_web_lg',
'es': 'es_core_news_lg',
'fr': 'fr_core_news_lg',
'it': 'it_core_news_lg',
'nl': 'nl_core_news_lg',
'pt': 'pt_core_news_lg'}
SPACY_MODELS = {'de': 'de_core_news_md',
'el': 'el_core_news_md',
'en': 'en_core_web_md',
'es': 'es_core_news_md',
'fr': 'fr_core_news_md',
'it': 'it_core_news_md',
'nl': 'nl_core_news_md',
'pt': 'pt_core_news_md'}
SPACY_MODELS_VERSION = os.environ.get('SPACY_MODELS_VERSION')
SPACY_VERSION = os.environ.get('SPACY_VERSION')