sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2024-12-26 21:34:18 +00:00
Code Issues Projects Releases Wiki Activity

Use smaller models

Browse Source
This commit is contained in:
Patrick Jentsch 2020-09-23 15:46:43 +02:00
parent 1ed42f68ad
commit 8afdfb13b2
4 changed files with 29 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitlab-ci.yml
View File

@ -3,16 +3,12 @@ image: docker:stable
services: services:
- docker:stable-dind - docker:stable-dind
variables:
DOCKER_DRIVER: overlay2
stages: stages:
- build - build
- push - push
.docker_setup: variables:
before_script: DOCKER_DRIVER: overlay2
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
.reg_setup: .reg_setup:
before_script: before_script:
@ -25,8 +21,8 @@ stages:
REG_VERSION: 0.16.1 REG_VERSION: 0.16.1
build_image: build_image:
extends: .docker_setup
script: script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker build -t $INTERMEDIATE_IMAGE_TAG . - docker build -t $INTERMEDIATE_IMAGE_TAG .
- docker push $INTERMEDIATE_IMAGE_TAG - docker push $INTERMEDIATE_IMAGE_TAG
stage: build stage: build
@ -37,11 +33,11 @@ build_image:
push_master: push_master:
extends: extends:
- .docker_setup
- .reg_setup - .reg_setup
only: only:
- master - master
script: script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $INTERMEDIATE_IMAGE_TAG - docker pull $INTERMEDIATE_IMAGE_TAG
- /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $INTERMEDIATE_IMAGE_TAG - /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $INTERMEDIATE_IMAGE_TAG
- docker tag $INTERMEDIATE_IMAGE_TAG $IMAGE_TAG - docker tag $INTERMEDIATE_IMAGE_TAG $IMAGE_TAG
@ -55,7 +51,6 @@ push_master:
push_other: push_other:
extends: extends:
- .docker_setup
- .reg_setup - .reg_setup
except: except:
- master - master
@ -63,6 +58,7 @@ push_other:
- branches - branches
- tags - tags
script: script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- docker pull $INTERMEDIATE_IMAGE_TAG - docker pull $INTERMEDIATE_IMAGE_TAG
- /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $INTERMEDIATE_IMAGE_TAG - /usr/local/bin/reg rm -d --auth-url $CI_REGISTRY -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $INTERMEDIATE_IMAGE_TAG
- docker tag $INTERMEDIATE_IMAGE_TAG $IMAGE_TAG - docker tag $INTERMEDIATE_IMAGE_TAG $IMAGE_TAG

16
Dockerfile
View File

@ -34,14 +34,14 @@ RUN apt-get install -y --no-install-recommends \
setuptools \ setuptools \
wheel \ wheel \
&& pip3 install "spacy==${SPACY_VERSION}" \ && pip3 install "spacy==${SPACY_VERSION}" \
&& python3 -m spacy download "de_core_news_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "de_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "el_core_news_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "el_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "en_core_web_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "en_core_web_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "es_core_news_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "es_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "fr_core_news_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "fr_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "it_core_news_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "it_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "nl_core_news_lg-${SPACY_MODELS_VERSION}" --direct \ && python3 -m spacy download "nl_core_news_md-${SPACY_MODELS_VERSION}" --direct \
&& python3 -m spacy download "pt_core_news_lg-${SPACY_MODELS_VERSION}" --direct && python3 -m spacy download "pt_core_news_md-${SPACY_MODELS_VERSION}" --direct
COPY nlp /usr/local/bin COPY nlp /usr/local/bin
COPY spacy-nlp /usr/local/bin COPY spacy-nlp /usr/local/bin

16
nlp
View File

@ -16,14 +16,14 @@ import os
import sys import sys
SPACY_MODELS = {'de': 'de_core_news_sm', SPACY_MODELS = {'de': 'de_core_news_md',
'el': 'el_core_news_sm', 'el': 'el_core_news_md',
'en': 'en_core_web_sm', 'en': 'en_core_web_md',
'es': 'es_core_news_sm', 'es': 'es_core_news_md',
'fr': 'fr_core_news_sm', 'fr': 'fr_core_news_md',
'it': 'it_core_news_sm', 'it': 'it_core_news_md',
'nl': 'nl_core_news_sm', 'nl': 'nl_core_news_md',
'pt': 'pt_core_news_sm'} 'pt': 'pt_core_news_md'}
def parse_args(): def parse_args():

16
spacy-nlp
View File

@ -9,14 +9,14 @@ import os
import spacy import spacy
import textwrap import textwrap
SPACY_MODELS = {'de': 'de_core_news_lg', SPACY_MODELS = {'de': 'de_core_news_md',
'el': 'el_core_news_lg', 'el': 'el_core_news_md',
'en': 'en_core_web_lg', 'en': 'en_core_web_md',
'es': 'es_core_news_lg', 'es': 'es_core_news_md',
'fr': 'fr_core_news_lg', 'fr': 'fr_core_news_md',
'it': 'it_core_news_lg', 'it': 'it_core_news_md',
'nl': 'nl_core_news_lg', 'nl': 'nl_core_news_md',
'pt': 'pt_core_news_lg'} 'pt': 'pt_core_news_md'}
SPACY_MODELS_VERSION = os.environ.get('SPACY_MODELS_VERSION') SPACY_MODELS_VERSION = os.environ.get('SPACY_MODELS_VERSION')
SPACY_VERSION = os.environ.get('SPACY_VERSION') SPACY_VERSION = os.environ.get('SPACY_VERSION')