pinthoz commited on
Commit
ed09cf3
·
verified ·
1 Parent(s): a6b1404

Fix isa plot

Browse files
Files changed (1) hide show
  1. attention_app/isa.py +5 -0
attention_app/isa.py CHANGED
@@ -9,6 +9,11 @@ try:
9
  except LookupError:
10
  nltk.download('punkt')
11
 
 
 
 
 
 
12
  def get_sentence_boundaries(text: str, tokens: List[str], tokenizer, inputs) -> Tuple[List[str], List[int]]:
13
  """
14
  Split text into sentences and map tokens to sentences.
 
9
  except LookupError:
10
  nltk.download('punkt')
11
 
12
+ try:
13
+ nltk.data.find('tokenizers/punkt_tab')
14
+ except LookupError:
15
+ nltk.download('punkt_tab')
16
+
17
  def get_sentence_boundaries(text: str, tokens: List[str], tokenizer, inputs) -> Tuple[List[str], List[int]]:
18
  """
19
  Split text into sentences and map tokens to sentences.