Spaces:
Sleeping
Sleeping
Fix isa plot
Browse files- attention_app/isa.py +5 -0
attention_app/isa.py
CHANGED
|
@@ -9,6 +9,11 @@ try:
|
|
| 9 |
except LookupError:
|
| 10 |
nltk.download('punkt')
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
def get_sentence_boundaries(text: str, tokens: List[str], tokenizer, inputs) -> Tuple[List[str], List[int]]:
|
| 13 |
"""
|
| 14 |
Split text into sentences and map tokens to sentences.
|
|
|
|
| 9 |
except LookupError:
|
| 10 |
nltk.download('punkt')
|
| 11 |
|
| 12 |
+
try:
|
| 13 |
+
nltk.data.find('tokenizers/punkt_tab')
|
| 14 |
+
except LookupError:
|
| 15 |
+
nltk.download('punkt_tab')
|
| 16 |
+
|
| 17 |
def get_sentence_boundaries(text: str, tokens: List[str], tokenizer, inputs) -> Tuple[List[str], List[int]]:
|
| 18 |
"""
|
| 19 |
Split text into sentences and map tokens to sentences.
|