# Copyright (c) 2005-2006 ActiveState Software Inc.
#
# Contributors:
#   Eric Promislow (EricP@ActiveState.com)

"""
Perl lexing support for codeintel/perlcile.py

Get all the lexed tokens from SilverCity, and then return them
on demand to the caller (usually a Perl pseudo-parser).

Usage:
import perl_lexer
lexer = lex_wrapper.Lexer(code)
while 1:
    tok = lexer.get_next_token()
    if tok[0] == EOF_STYLE:
        break;
    # tok is an array of (style, text, start-col, start-line, end-col, end-line)
    # column and line numbers are all zero-based.
"""

import copy
import re
import sys
import string

import SilverCity
from SilverCity import Perl, ScintillaConstants
from language.legacy.common import shared_lexer
from language.legacy.common.shared_lexer import EOF_STYLE

pod_markings = re.compile('^=(?:head|item|cut)', re.M)

class PerlLexerClassifier:
    """ This classifier is similar to the parser-level classifier, but
    it works on the SilverCity "raw" tokens as opposed to the
    tokens that get created by the lexer layer.  There should be some
    folding though."""

    def is_comment(self, ttype):
        return ttype in (ScintillaConstants.SCE_PL_COMMENTLINE,
                         ScintillaConstants.SCE_PL_POD)

    @property
    def style_comment(self):
        return ScintillaConstants.SCE_PL_COMMENTLINE

    @property
    def style_default(self):
        return ScintillaConstants.SCE_PL_DEFAULT

    @property
    def style_operator(self):
        return ScintillaConstants.SCE_PL_OPERATOR


class _CommonLexer(shared_lexer.Lexer):
    def __init__(self):
        shared_lexer.Lexer.__init__(self)
        self.q = []
        self.multi_char_ops = self.build_dict('-> ++ -- ** =~ !~ << >> <= >= == != <=> && || // ... .. => <<= >>= &&= ||= ~*= /= %= += -= .= &= |= ^= ::')

class PerlLexer(_CommonLexer):
    def __init__(self, code, provide_full_docs=True):
        _CommonLexer.__init__(self)
        self.q = []
        self.classifier = PerlLexerClassifier()
        self._provide_full_docs = provide_full_docs
        Perl.PerlLexer().tokenize_by_style(code, self._fix_token_list)
        self.prepare_token_list_for_use()
        self.string_types = [ScintillaConstants.SCE_PL_STRING,
                         ScintillaConstants.SCE_PL_CHARACTER,
                         ScintillaConstants.SCE_PL_HERE_Q,
                         ScintillaConstants.SCE_PL_HERE_QQ,
                         ScintillaConstants.SCE_PL_STRING_QW,
                         ScintillaConstants.SCE_PL_STRING_Q,
                         ScintillaConstants.SCE_PL_STRING_QQ,
                         ScintillaConstants.SCE_PL_STRING_QX
                         ]

    def _fix_token_list(self, **tok):
        """ SilverCity doesn't know much about Perl, and breaks in two ways:
        1. It doesn't know how to separate sequences of characters into
        separate tokens.
        2. It doesn't know how to map DATASECTIONs into POD sequences.

        It's easier to do this once before processing tokens individually.

        This should all be done in silvercity.  Doing this has leaked the
        whole silvercity abstraction into this module, and it doesn't
        belong here.  This routine works with SilverCity tokens, not
        shared_lexer Tokens.
        """
        if tok['start_column'] > shared_lexer.MAX_REASONABLE_LIMIT:
            return
        ttype = tok['style']
        tval = tok['text']
        if ttype in (ScintillaConstants.SCE_PL_OPERATOR,
                     ScintillaConstants.SCE_PL_VARIABLE_INDEXER) and len(tok['text']) > 1:
            # Scineplex doesn't know how to split some sequences of styled characters
            # into syntactically different tokens, so we do it here.
            # A sequence of characters might need to be split into more than one token.
            # Push all but the last token on the pending block.
            self.append_split_tokens(tok, self.multi_char_ops)
        elif ttype == ScintillaConstants.SCE_PL_IDENTIFIER:
            tok['text'] = tok['text'].strip()
            self.complete_token_push(tok)
        elif (not self._provide_full_docs) and \
                ttype in (ScintillaConstants.SCE_PL_DATASECTION,
                          ScintillaConstants.SCE_PL_POD):
            pass
        elif ttype == ScintillaConstants.SCE_PL_DATASECTION:
            if pod_markings.search(tval):
                # putback (KWD package), (ID main), (OP ;), (POD this)
                col = tok['start_column']
                for new_vals in ((ScintillaConstants.SCE_PL_WORD, "package"),
                                 (ScintillaConstants.SCE_PL_IDENTIFIER, "main"),
                                 (ScintillaConstants.SCE_PL_OPERATOR, ";")):
                    new_type, new_text = new_vals
                    new_tok = copy.copy(tok)
                    new_tok['text'] = new_text
                    new_tok['style'] = new_type
                    new_tok['start_column'] = col
                    new_tok['end_column'] = col + len(new_text) - 1
                    col = new_tok['end_column'] + 1
                    self.complete_token_push(new_tok)
                tok['style'] = ScintillaConstants.SCE_PL_POD
                tok['text'] = tval;
                tok['start_column'] = col
                if tok['start_line'] == tok['end_line']:
                    tok['end_column'] = tok['start_line'] + len(tok['text']) - 1
                self.complete_token_push(tok)
            else:
                # End of the queue => EOF
                pass
        else:
            self.complete_token_push(tok)

class PerlMultiLangLexer(_CommonLexer):
    def __init__(self, token_source):
        _CommonLexer.__init__(self)
        self.csl_tokens = []
        self.css_tokens = []
        # http://www.mozilla.org/js/language/grammar14.html
        self.js_multi_char_ops = self.build_dict('++ -- << >> >>> <= >= == != === !== && || *= /= %= += -= <<= >>= >>>= &= ^= |=')
        self.string_types = [ScintillaConstants.SCE_UDL_SSL_STRING
                ]
        self.classifier = shared_lexer.UDLLexerClassifier()
        self._contains_ssl = False
        self._build_tokens(token_source)
        self.prepare_token_list_for_use()

    def _build_tokens(self, token_source):
        while True:
            try:
                tok = token_source.next()
                self._fix_token_list(tok)
            except StopIteration:
                break

    def _fix_token_list(self, tok):
        """See perl_lexer.py for details on what this routine does."""
        ttype = tok['style']
        tval = tok['text']

        if self.is_udl_csl_family(ttype):
            # Don't adjust the line numbers of CSL tokens this time,
            # have it done later.
            if ttype == ScintillaConstants.SCE_UDL_CSL_OPERATOR and len(tval) > 1:
                # Point the token splitter to the correct token queue
                self.append_split_tokens(tok, self.js_multi_char_ops,
                                         adjust_line=False,
                                         dest_q=self.csl_tokens)
            else:
                self.complete_token_push(tok, adjust_line=False,
                                         dest_q=self.csl_tokens)
        elif self.is_udl_ssl_family(ttype):
            if tok['style'] == ScintillaConstants.SCE_UDL_SSL_OPERATOR and len(tok['text']) > 1:
                self.append_split_tokens(tok, self.multi_char_ops)
            else:
                self.complete_token_push(tok)
            self._contains_ssl = True
        elif self.is_udl_css_family(ttype):
            self.complete_token_push(tok, adjust_line=False,
                                     dest_q=self.css_tokens)
        # See comment in RubyMultiLangLexer._fix_token_list
        # on why we probably don't need this code.
        #elif self.is_udl_tpl_family(ttype):
        #    self._contains_ssl = True

    def get_csl_tokens(self):
        return self.csl_tokens

    def get_css_tokens(self):
        return self.css_tokens

    def has_perl_code(self):
        return self._contains_ssl

def provide_sample_code():
    return r"""use LWP::UserAgent;

# full-line comment
# comment at start of line
 # comment at col 1
  # comment at col 2

{
package Foo;

sub new {
   my ($class, %args) = @_;
   my $self = {count => 0, list = [] };
   bless $self, (ref $class || $class);
}

sub m {
    my ($self, $arg) = @_;
    push @{$self->{list}}, $arg;
    $self->{count} += 1;
    $arg;
}

sub no_paren {
  my ($a, $b,
     $c) = $_;
  print "blah";
}

sub our_generate { my $self = shift; my $file_info = shift;
  $self->{info} = info;
  $self->{files} = [];
  $self->{classes} = [];
  $self->{hyperlinks} = {};
  #comment on what test_fn does
  # more...

  sub test_fn {
        my ($a, $b, $c, $d, $e) = @_;
        $b |= 'val1';
        $c |= f(3);
       print "nothing\n";   # end-of-line comment
       print qq(percent string\n);
    }

   }
   }
"""


if __name__ == "__main__":
    shared_lexer.main(sys.argv, provide_sample_code, PerlLexer)
