+++ /dev/null
-# -*- coding: utf-8 -*-
-# Natural Language Toolkit: Interface to MaltParser
-#
-# Author: Dan Garrette <dhgarrette@gmail.com>
-# Contributor: Liling Tan, Mustufain, osamamukhtar11
-#
-# Copyright (C) 2001-2019 NLTK Project
-# URL: <http://nltk.org/>
-# For license information, see LICENSE.TXT
-
-from __future__ import print_function, unicode_literals
-
-import os
-import sys
-import tempfile
-import subprocess
-import inspect
-
-from six import text_type
-
-from nltk.data import ZipFilePathPointer
-from nltk.internals import find_dir, find_file, find_jars_within_path
-
-from nltk.parse.api import ParserI
-from nltk.parse.dependencygraph import DependencyGraph
-from nltk.parse.util import taggedsents_to_conll
-
-
-def malt_regex_tagger():
- from nltk.tag import RegexpTagger
-
- _tagger = RegexpTagger(
- [
- (r'\.$', '.'),
- (r'\,$', ','),
- (r'\?$', '?'), # fullstop, comma, Qmark
- (r'\($', '('),
- (r'\)$', ')'), # round brackets
- (r'\[$', '['),
- (r'\]$', ']'), # square brackets
- (r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
- (r'(The|the|A|a|An|an)$', 'DT'), # articles
- (r'(He|he|She|she|It|it|I|me|Me|You|you)$', 'PRP'), # pronouns
- (r'(His|his|Her|her|Its|its)$', 'PRP$'), # possesive
- (r'(my|Your|your|Yours|yours)$', 'PRP$'), # possesive
- (r'(on|On|in|In|at|At|since|Since)$', 'IN'), # time prepopsitions
- (r'(for|For|ago|Ago|before|Before)$', 'IN'), # time prepopsitions
- (r'(till|Till|until|Until)$', 'IN'), # time prepopsitions
- (r'(by|By|beside|Beside)$', 'IN'), # space prepopsitions
- (r'(under|Under|below|Below)$', 'IN'), # space prepopsitions
- (r'(over|Over|above|Above)$', 'IN'), # space prepopsitions
- (r'(across|Across|through|Through)$', 'IN'), # space prepopsitions
- (r'(into|Into|towards|Towards)$', 'IN'), # space prepopsitions
- (r'(onto|Onto|from|From)$', 'IN'), # space prepopsitions
- (r'.*able$', 'JJ'), # adjectives
- (r'.*ness$', 'NN'), # nouns formed from adjectives
- (r'.*ly$', 'RB'), # adverbs
- (r'.*s$', 'NNS'), # plural nouns
- (r'.*ing$', 'VBG'), # gerunds
- (r'.*ed$', 'VBD'), # past tense verbs
- (r'.*', 'NN'), # nouns (default)
- ]
- )
- return _tagger.tag
-
-
-def find_maltparser(parser_dirname):
- """
- A module to find MaltParser .jar file and its dependencies.
- """
- if os.path.exists(parser_dirname): # If a full path is given.
- _malt_dir = parser_dirname
- else: # Try to find path to maltparser directory in environment variables.
- _malt_dir = find_dir(parser_dirname, env_vars=('MALT_PARSER',))
- # Checks that that the found directory contains all the necessary .jar
- malt_dependencies = ['', '', '']
- _malt_jars = set(find_jars_within_path(_malt_dir))
- _jars = set(os.path.split(jar)[1] for jar in _malt_jars)
- malt_dependencies = set(['log4j.jar', 'libsvm.jar', 'liblinear-1.8.jar'])
-
- assert malt_dependencies.issubset(_jars)
- assert any(
- filter(lambda i: i.startswith('maltparser-') and i.endswith('.jar'), _jars)
- )
- return list(_malt_jars)
-
-
-def find_malt_model(model_filename):
- """
- A module to find pre-trained MaltParser model.
- """
- if model_filename is None:
- return 'malt_temp.mco'
- elif os.path.exists(model_filename): # If a full path is given.
- return model_filename
- else: # Try to find path to malt model in environment variables.
- return find_file(model_filename, env_vars=('MALT_MODEL',), verbose=False)
-
-
-class MaltParser(ParserI):
- """
- A class for dependency parsing with MaltParser. The input is the paths to:
- - a maltparser directory
- - (optionally) the path to a pre-trained MaltParser .mco model file
- - (optionally) the tagger to use for POS tagging before parsing
- - (optionally) additional Java arguments
-
- Example:
- >>> from nltk.parse import malt
- >>> # With MALT_PARSER and MALT_MODEL environment set.
- >>> mp = malt.MaltParser('maltparser-1.7.2', 'engmalt.linear-1.7.mco') # doctest: +SKIP
- >>> mp.parse_one('I shot an elephant in my pajamas .'.split()).tree() # doctest: +SKIP
- (shot I (elephant an) (in (pajamas my)) .)
- >>> # Without MALT_PARSER and MALT_MODEL environment.
- >>> mp = malt.MaltParser('/home/user/maltparser-1.7.2/', '/home/user/engmalt.linear-1.7.mco') # doctest: +SKIP
- >>> mp.parse_one('I shot an elephant in my pajamas .'.split()).tree() # doctest: +SKIP
- (shot I (elephant an) (in (pajamas my)) .)
- """
-
- def __init__(
- self,
- parser_dirname,
- model_filename=None,
- tagger=None,
- additional_java_args=None,
- ):
- """
- An interface for parsing with the Malt Parser.
-
- :param parser_dirname: The path to the maltparser directory that
- contains the maltparser-1.x.jar
- :type parser_dirname: str
- :param model_filename: The name of the pre-trained model with .mco file
- extension. If provided, training will not be required.
- (see http://www.maltparser.org/mco/mco.html and
- see http://www.patful.com/chalk/node/185)
- :type model_filename: str
- :param tagger: The tagger used to POS tag the raw string before
- formatting to CONLL format. It should behave like `nltk.pos_tag`
- :type tagger: function
- :param additional_java_args: This is the additional Java arguments that
- one can use when calling Maltparser, usually this is the heapsize
- limits, e.g. `additional_java_args=['-Xmx1024m']`
- (see http://goo.gl/mpDBvQ)
- :type additional_java_args: list
- """
-
- # Find all the necessary jar files for MaltParser.
- self.malt_jars = find_maltparser(parser_dirname)
- # Initialize additional java arguments.
- self.additional_java_args = (
- additional_java_args if additional_java_args is not None else []
- )
- # Initialize model.
- self.model = find_malt_model(model_filename)
- self._trained = self.model != 'malt_temp.mco'
- # Set the working_dir parameters i.e. `-w` from MaltParser's option.
- self.working_dir = tempfile.gettempdir()
- # Initialize POS tagger.
- self.tagger = tagger if tagger is not None else malt_regex_tagger()
-
- def parse_tagged_sents(self, sentences, verbose=False, top_relation_label='null'):
- """
- Use MaltParser to parse multiple POS tagged sentences. Takes multiple
- sentences where each sentence is a list of (word, tag) tuples.
- The sentences must have already been tokenized and tagged.
-
- :param sentences: Input sentences to parse
- :type sentence: list(list(tuple(str, str)))
- :return: iter(iter(``DependencyGraph``)) the dependency graph
- representation of each sentence
- """
- if not self._trained:
- raise Exception("Parser has not been trained. Call train() first.")
-
- with tempfile.NamedTemporaryFile(
- prefix='malt_input.conll.', dir=self.working_dir, mode='w', delete=False
- ) as input_file:
- with tempfile.NamedTemporaryFile(
- prefix='malt_output.conll.',
- dir=self.working_dir,
- mode='w',
- delete=False,
- ) as output_file:
- # Convert list of sentences to CONLL format.
- for line in taggedsents_to_conll(sentences):
- input_file.write(text_type(line))
- input_file.close()
-
- # Generate command to run maltparser.
- cmd = self.generate_malt_command(
- input_file.name, output_file.name, mode="parse"
- )
-
- # This is a maltparser quirk, it needs to be run
- # where the model file is. otherwise it goes into an awkward
- # missing .jars or strange -w working_dir problem.
- _current_path = os.getcwd() # Remembers the current path.
- try: # Change to modelfile path
- os.chdir(os.path.split(self.model)[0])
- except:
- pass
- ret = self._execute(cmd, verbose) # Run command.
- os.chdir(_current_path) # Change back to current path.
-
- if ret is not 0:
- raise Exception(
- "MaltParser parsing (%s) failed with exit "
- "code %d" % (' '.join(cmd), ret)
- )
-
- # Must return iter(iter(Tree))
- with open(output_file.name) as infile:
- for tree_str in infile.read().split('\n\n'):
- yield (
- iter(
- [
- DependencyGraph(
- tree_str, top_relation_label=top_relation_label
- )
- ]
- )
- )
-
- os.remove(input_file.name)
- os.remove(output_file.name)
-
- def parse_sents(self, sentences, verbose=False, top_relation_label='null'):
- """
- Use MaltParser to parse multiple sentences.
- Takes a list of sentences, where each sentence is a list of words.
- Each sentence will be automatically tagged with this
- MaltParser instance's tagger.
-
- :param sentences: Input sentences to parse
- :type sentence: list(list(str))
- :return: iter(DependencyGraph)
- """
- tagged_sentences = (self.tagger(sentence) for sentence in sentences)
- return self.parse_tagged_sents(
- tagged_sentences, verbose, top_relation_label=top_relation_label
- )
-
- def generate_malt_command(self, inputfilename, outputfilename=None, mode=None):
- """
- This function generates the maltparser command use at the terminal.
-
- :param inputfilename: path to the input file
- :type inputfilename: str
- :param outputfilename: path to the output file
- :type outputfilename: str
- """
-
- cmd = ['java']
- cmd += self.additional_java_args # Adds additional java arguments
- # Joins classpaths with ";" if on Windows and on Linux/Mac use ":"
- classpaths_separator = ';' if sys.platform.startswith('win') else ':'
- cmd += [
- '-cp',
- classpaths_separator.join(self.malt_jars),
- ] # Adds classpaths for jars
- cmd += ['org.maltparser.Malt'] # Adds the main function.
-
- # Adds the model file.
- if os.path.exists(self.model): # when parsing
- cmd += ['-c', os.path.split(self.model)[-1]]
- else: # when learning
- cmd += ['-c', self.model]
-
- cmd += ['-i', inputfilename]
- if mode == 'parse':
- cmd += ['-o', outputfilename]
- cmd += ['-m', mode] # mode use to generate parses.
- return cmd
-
- @staticmethod
- def _execute(cmd, verbose=False):
- output = None if verbose else subprocess.PIPE
- p = subprocess.Popen(cmd, stdout=output, stderr=output)
- return p.wait()
-
- def train(self, depgraphs, verbose=False):
- """
- Train MaltParser from a list of ``DependencyGraph`` objects
-
- :param depgraphs: list of ``DependencyGraph`` objects for training input data
- :type depgraphs: DependencyGraph
- """
-
- # Write the conll_str to malt_train.conll file in /tmp/
- with tempfile.NamedTemporaryFile(
- prefix='malt_train.conll.', dir=self.working_dir, mode='w', delete=False
- ) as input_file:
- input_str = '\n'.join(dg.to_conll(10) for dg in depgraphs)
- input_file.write(text_type(input_str))
- # Trains the model with the malt_train.conll
- self.train_from_file(input_file.name, verbose=verbose)
- # Removes the malt_train.conll once training finishes.
- os.remove(input_file.name)
-
- def train_from_file(self, conll_file, verbose=False):
- """
- Train MaltParser from a file
- :param conll_file: str for the filename of the training input data
- :type conll_file: str
- """
-
- # If conll_file is a ZipFilePathPointer,
- # then we need to do some extra massaging
- if isinstance(conll_file, ZipFilePathPointer):
- with tempfile.NamedTemporaryFile(
- prefix='malt_train.conll.', dir=self.working_dir, mode='w', delete=False
- ) as input_file:
- with conll_file.open() as conll_input_file:
- conll_str = conll_input_file.read()
- input_file.write(text_type(conll_str))
- return self.train_from_file(input_file.name, verbose=verbose)
-
- # Generate command to run maltparser.
- cmd = self.generate_malt_command(conll_file, mode="learn")
- ret = self._execute(cmd, verbose)
- if ret != 0:
- raise Exception(
- "MaltParser training (%s) failed with exit "
- "code %d" % (' '.join(cmd), ret)
- )
- self._trained = True
-
-
-if __name__ == '__main__':
- '''
- A demostration function to show how NLTK users can use the malt parser API.
-
- >>> from nltk import pos_tag
- >>> assert 'MALT_PARSER' in os.environ, str(
- ... "Please set MALT_PARSER in your global environment, e.g.:\n"
- ... "$ export MALT_PARSER='/home/user/maltparser-1.7.2/'")
- >>>
- >>> assert 'MALT_MODEL' in os.environ, str(
- ... "Please set MALT_MODEL in your global environment, e.g.:\n"
- ... "$ export MALT_MODEL='/home/user/engmalt.linear-1.7.mco'")
- >>>
- >>> _dg1_str = str("1 John _ NNP _ _ 2 SUBJ _ _\n"
- ... "2 sees _ VB _ _ 0 ROOT _ _\n"
- ... "3 a _ DT _ _ 4 SPEC _ _\n"
- ... "4 dog _ NN _ _ 2 OBJ _ _\n"
- ... "5 . _ . _ _ 2 PUNCT _ _\n")
- >>>
- >>>
- >>> _dg2_str = str("1 John _ NNP _ _ 2 SUBJ _ _\n"
- ... "2 walks _ VB _ _ 0 ROOT _ _\n"
- ... "3 . _ . _ _ 2 PUNCT _ _\n")
- >>> dg1 = DependencyGraph(_dg1_str)
- >>> dg2 = DependencyGraph(_dg2_str)
- >>> # Initialize a MaltParser object
- >>> parser_dirname = 'maltparser-1.7.2'
- >>> mp = MaltParser(parser_dirname=parser_dirname)
- >>>
- >>> # Trains a model.
- >>> mp.train([dg1,dg2], verbose=False)
- >>> sent1 = ['John','sees','Mary', '.']
- >>> sent2 = ['John', 'walks', 'a', 'dog', '.']
- >>>
- >>> # Parse a single sentence.
- >>> parsed_sent1 = mp.parse_one(sent1)
- >>> parsed_sent2 = mp.parse_one(sent2)
- >>> print (parsed_sent1.tree())
- (sees John Mary .)
- >>> print (parsed_sent2.tree())
- (walks John (dog a) .)
- >>>
- >>> # Parsing multiple sentences.
- >>> sentences = [sent1,sent2]
- >>> parsed_sents = mp.parse_sents(sentences)
- >>> print(next(next(parsed_sents)).tree())
- (sees John Mary .)
- >>> print(next(next(parsed_sents)).tree())
- (walks John (dog a) .)
- >>>
- >>> # Initialize a MaltParser object with an English pre-trained model.
- >>> parser_dirname = 'maltparser-1.7.2'
- >>> model_name = 'engmalt.linear-1.7.mco'
- >>> mp = MaltParser(parser_dirname=parser_dirname, model_filename=model_name, tagger=pos_tag)
- >>> sent1 = 'I shot an elephant in my pajamas .'.split()
- >>> sent2 = 'Time flies like banana .'.split()
- >>> # Parse a single sentence.
- >>> print(mp.parse_one(sent1).tree())
- (shot I (elephant an) (in (pajamas my)) .)
- # Parsing multiple sentences
- >>> sentences = [sent1,sent2]
- >>> parsed_sents = mp.parse_sents(sentences)
- >>> print(next(next(parsed_sents)).tree())
- (shot I (elephant an) (in (pajamas my)) .)
- >>> print(next(next(parsed_sents)).tree())
- (flies Time (like banana) .)
- '''
- import doctest
-
- doctest.testmod()
projects / platform / core / uifw / nlp.git / commitdiff