src/Pure/Syntax/parser.ML
changeset 23909 6e4fba2ea7d0
parent 22573 2ac646ab2f6c
child 23941 6234185a2e2e
equal deleted inserted replaced
23908:edca7f581c09 23909:6e4fba2ea7d0
    56 val UnknownStart = EndToken;       (*productions for which no starting token is
    56 val UnknownStart = EndToken;       (*productions for which no starting token is
    57                                      known yet are associated with this token*)
    57                                      known yet are associated with this token*)
    58 
    58 
    59 (* get all NTs that are connected with a list of NTs
    59 (* get all NTs that are connected with a list of NTs
    60    (used for expanding chain list)*)
    60    (used for expanding chain list)*)
    61 fun connected_with _ [] relatives = relatives
    61 fun connected_with _ ([]: nt_tag list) relatives = relatives
    62   | connected_with chains (root :: roots) relatives =
    62   | connected_with chains (root :: roots) relatives =
    63     let val branches = subtract (op =) relatives (these (AList.lookup (op =) chains root));
    63     let val branches = subtract (op =) relatives (these (AList.lookup (op =) chains root));
    64     in connected_with chains (branches @ roots) (branches @ relatives) end;
    64     in connected_with chains (branches @ roots) (branches @ relatives) end;
    65 
    65 
    66 (* convert productions to grammar;
    66 (* convert productions to grammar;
    99                                        else (to, new_tks) :: added)
    99                                        else (to, new_tks) :: added)
   100                 end;
   100                 end;
   101 
   101 
   102             val tos = connected_with chains' [lhs] [lhs];
   102             val tos = connected_with chains' [lhs] [lhs];
   103         in (copy_lookahead tos [],
   103         in (copy_lookahead tos [],
   104             (if member (op =) lambdas lhs then tos else []) union lambdas)
   104             gen_union (op =) (if member (op =) lambdas lhs then tos else [], lambdas))
   105         end;
   105         end;
   106 
   106 
   107       (*test if new production can produce lambda
   107       (*test if new production can produce lambda
   108         (rhs must either be empty or only consist of lambda NTs)*)
   108         (rhs must either be empty or only consist of lambda NTs)*)
   109       val (new_lambda, lambdas') =
   109       val (new_lambda, lambdas') =
   110         if forall (fn (Nonterminal (id, _)) => member (op =) lambdas' id
   110         if forall (fn (Nonterminal (id, _)) => member (op =) lambdas' id
   111                     | (Terminal _) => false) rhs then
   111                     | (Terminal _) => false) rhs then
   112           (true, lambdas' union (connected_with chains' [lhs] [lhs]))
   112           (true, gen_union (op =) (lambdas', connected_with chains' [lhs] [lhs]))
   113         else
   113         else
   114           (false, lambdas');
   114           (false, lambdas');
   115 
   115 
   116       (*list optional terminal and all nonterminals on which the lookahead
   116       (*list optional terminal and all nonterminals on which the lookahead
   117         of a production depends*)
   117         of a production depends*)
   153                           ((if is_some tk then [the tk] else []) @
   153                           ((if is_some tk then [the tk] else []) @
   154                             Library.foldl token_union ([], map starts_for_nt nts));
   154                             Library.foldl token_union ([], map starts_for_nt nts));
   155 
   155 
   156                         val added_tks' = token_union (new_tks, added_tks);
   156                         val added_tks' = token_union (new_tks, added_tks);
   157 
   157 
   158                         val nt_dependencies' = nts union nt_dependencies;
   158                         val nt_dependencies' = gen_union (op =) (nts, nt_dependencies);
   159 
   159 
   160                         (*associate production with new starting tokens*)
   160                         (*associate production with new starting tokens*)
   161                         fun copy [] nt_prods = nt_prods
   161                         fun copy ([]: token option list) nt_prods = nt_prods
   162                           | copy (tk :: tks) nt_prods =
   162                           | copy (tk :: tks) nt_prods =
   163                             let val old_prods = these (AList.lookup (op =) nt_prods tk);
   163                             let val old_prods = these (AList.lookup (op =) nt_prods tk);
   164 
   164 
   165                                 val prods' = p :: old_prods;
   165                                 val prods' = p :: old_prods;
   166                             in nt_prods
   166                             in nt_prods
   276                             if member (op =) tk_prods new_prod then (tk_prods, false)
   276                             if member (op =) tk_prods new_prod then (tk_prods, false)
   277                             else (new_prod :: tk_prods, true)
   277                             else (new_prod :: tk_prods, true)
   278                           else (new_prod :: tk_prods, true);
   278                           else (new_prod :: tk_prods, true);
   279 
   279 
   280                         val prods' = prods
   280                         val prods' = prods
   281                                      |> is_new' ? AList.update (op =) (tk, tk_prods');
   281                           |> is_new' ? AList.update (op =) (tk: token option, tk_prods');
   282                     in store tks prods' (is_new orelse is_new') end;
   282                     in store tks prods' (is_new orelse is_new') end;
   283 
   283 
   284                 val (nt_prods', prod_count', changed) =
   284                 val (nt_prods', prod_count', changed) =
   285                   if nt = lhs then store opt_starts nt_prods false
   285                   if nt = lhs then store opt_starts nt_prods false
   286                               else (nt_prods, prod_count, false);
   286                               else (nt_prods, prod_count, false);
   311 
   311 
   312                 (*copy productions whose lookahead depends on changed_nt;
   312                 (*copy productions whose lookahead depends on changed_nt;
   313                   if key = SOME UnknownToken then tk_prods is used to hold
   313                   if key = SOME UnknownToken then tk_prods is used to hold
   314                   the productions not copied*)
   314                   the productions not copied*)
   315                 fun update_prods [] result = result
   315                 fun update_prods [] result = result
   316                   | update_prods ((p as (rhs, _, _)) :: ps)
   316                   | update_prods ((p as (rhs, _: string, _: nt_tag)) :: ps)
   317                       (tk_prods, nt_prods) =
   317                       (tk_prods, nt_prods) =
   318                     let
   318                     let
   319                       (*lookahead dependency for production*)
   319                       (*lookahead dependency for production*)
   320                       val (tk, depends) = lookahead_dependency lambdas' rhs [];
   320                       val (tk, depends) = lookahead_dependency lambdas' rhs [];
   321 
   321 
   327                       val lambda = length depends > 1 orelse
   327                       val lambda = length depends > 1 orelse
   328                                    not (null depends) andalso is_some tk
   328                                    not (null depends) andalso is_some tk
   329                                    andalso member (op =) new_tks (the tk);
   329                                    andalso member (op =) new_tks (the tk);
   330 
   330 
   331                       (*associate production with new starting tokens*)
   331                       (*associate production with new starting tokens*)
   332                       fun copy [] nt_prods = nt_prods
   332                       fun copy ([]: token list) nt_prods = nt_prods
   333                         | copy (tk :: tks) nt_prods =
   333                         | copy (tk :: tks) nt_prods =
   334                           let
   334                           let
   335                             val tk_prods = (these o AList.lookup (op =) nt_prods) (SOME tk);
   335                             val tk_prods = (these o AList.lookup (op =) nt_prods) (SOME tk);
   336 
   336 
   337                             val tk_prods' =
   337                             val tk_prods' =
   412     fun pretty_nt (name, tag) =
   412     fun pretty_nt (name, tag) =
   413       let
   413       let
   414         fun prod_of_chain from_ = ([Nonterminal (from_, ~1)], "", ~1);
   414         fun prod_of_chain from_ = ([Nonterminal (from_, ~1)], "", ~1);
   415 
   415 
   416         val nt_prods =
   416         val nt_prods =
   417           Library.foldl (op union) ([], map snd (snd (Array.sub (prods, tag)))) @
   417           Library.foldl (gen_union op =) ([], map snd (snd (Array.sub (prods, tag)))) @
   418           map prod_of_chain ((these o AList.lookup (op =) chains) tag);
   418           map prod_of_chain ((these o AList.lookup (op =) chains) tag);
   419       in map (pretty_prod name) nt_prods end;
   419       in map (pretty_prod name) nt_prods end;
   420 
   420 
   421   in maps pretty_nt taglist end;
   421   in maps pretty_nt taglist end;
   422 
   422 
   429                        prods = Array.array (0, (([], []), []))};
   429                        prods = Array.array (0, (([], []), []))};
   430 
   430 
   431 
   431 
   432 (*Invert list of chain productions*)
   432 (*Invert list of chain productions*)
   433 fun inverse_chains [] result = result
   433 fun inverse_chains [] result = result
   434   | inverse_chains ((root, branches) :: cs) result =
   434   | inverse_chains ((root, branches: nt_tag list) :: cs) result =
   435     let fun add [] result = result
   435     let fun add ([]: nt_tag list) result = result
   436           | add (id :: ids) result =
   436           | add (id :: ids) result =
   437             let val old = (these o AList.lookup (op =) result) id;
   437             let val old = (these o AList.lookup (op =) result) id;
   438             in add ids (AList.update (op =) (id, root :: old) result) end;
   438             in add ids (AList.update (op =) (id, root :: old) result) end;
   439     in inverse_chains cs (add branches result) end;
   439     in inverse_chains cs (add branches result) end;
   440 
   440 
   561 
   561 
   562     (*convert prods2 array to productions*)
   562     (*convert prods2 array to productions*)
   563     fun process_nt ~1 result = result
   563     fun process_nt ~1 result = result
   564       | process_nt nt result =
   564       | process_nt nt result =
   565         let
   565         let
   566           val nt_prods = Library.foldl (op union)
   566           val nt_prods = Library.foldl (gen_union op =)
   567                              ([], map snd (snd (Array.sub (prods2, nt))));
   567                              ([], map snd (snd (Array.sub (prods2, nt))));
   568           val lhs_tag = convert_tag nt;
   568           val lhs_tag = convert_tag nt;
   569 
   569 
   570           (*convert tags in rhs*)
   570           (*convert tags in rhs*)
   571           fun process_rhs [] result = result
   571           fun process_rhs [] result = result
   834                       val (_, nt_prods) = Array.sub (prods, nt);
   834                       val (_, nt_prods) = Array.sub (prods, nt);
   835 
   835 
   836                       val chained = map (fn nt => (nt, minPrec))
   836                       val chained = map (fn nt => (nt, minPrec))
   837                                         ((these o AList.lookup (op =) chains) nt);
   837                                         ((these o AList.lookup (op =) chains) nt);
   838                   in get_starts (chained @ nts)
   838                   in get_starts (chained @ nts)
   839                                 ((get nt_prods []) union result)
   839                                 (gen_union (op =) (get nt_prods [], result))
   840                   end;
   840                   end;
   841 
   841 
   842               val nts =
   842               val nts =
   843                 map_filter (fn (_, _, _, Nonterminal (a, prec) :: _, _, _) =>
   843                 map_filter (fn (_, _, _, Nonterminal (a, prec) :: _, _, _) =>
   844                            SOME (a, prec) | _ => NONE)
   844                            SOME (a, prec) | _ => NONE)