sporada/nlp
1
0
Fork 0
mirror of https://gitlab.ub.uni-bielefeld.de/sfb1288inf/nlp.git synced 2024-12-26 22:24:17 +00:00
Code Issues Projects Releases Wiki Activity

Fix problem where encoding is not set

Browse Source
This commit is contained in:
Patrick Jentsch 2021-06-22 12:46:01 +02:00
parent fd39246e4b
commit 5139fd9727
1 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spacy-nlp
View File

@ -27,14 +27,12 @@ parser.add_argument('-c', '--check-encoding',
help='Check encoding of the input file, UTF-8 is used instead') # noqa help='Check encoding of the input file, UTF-8 is used instead') # noqa
args = parser.parse_args() args = parser.parse_args()
if args.check_encoding:
with open(args.input, "rb") as text_file: with open(args.input, "rb") as text_file:
if args.check_encoding: if args.check_encoding:
encoding = chardet.detect(text_file.read())['encoding'] encoding = chardet.detect(text_file.read())['encoding']
else: else:
encoding = 'utf-8' encoding = 'utf-8'
text_file.seek(0)
with open(args.input, "rb") as text_file:
text_md5 = hashlib.md5() text_md5 = hashlib.md5()
for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''): for chunk in iter(lambda: text_file.read(128 * text_md5.block_size), b''):
text_md5.update(chunk) text_md5.update(chunk)