Handle C++20 module partitions in scanner

7 files changed, 176 insertions, 63 deletions

diff --git a/libbuild2/cc/lexer+basics.test.testscript b/libbuild2/cc/lexer+basics.test.testscript
new file mode 100644
index 0000000..65c6edf
--- /dev/null
+++ b/libbuild2/cc/lexer+basics.test.testscript
@@ -0,0 +1,14 @@
+# file      : libbuild2/cc/lexer+basics.test.testscript
+# license   : MIT; see accompanying LICENSE file
+
+# Test lexer basics (token recognition, etc)
+#
+
+# colon-scope
+#
+$* <<EOI >>EOO
+:::
+EOI
+'::'
+':'
+EOO
diff --git a/libbuild2/cc/lexer.cxx b/libbuild2/cc/lexer.cxx
index 123a41e..beeb970 100644
--- a/libbuild2/cc/lexer.cxx
+++ b/libbuild2/cc/lexer.cxx
@@ -377,9 +377,13 @@ namespace build2
             xchar p (peek ());
 
             if (p == ':')
+            {
               geth (p);
+              t.type = type::scope;
+            }
+            else
+              t.type = type::colon;
 
-            t.type = type::punctuation;
             return;
           }
           // Number (and also .<N> above).
@@ -1158,6 +1162,8 @@ namespace build2
       {
       case type::dot:         o << "'.'";                   break;
       case type::semi:        o << "';'";                   break;
+      case type::colon:       o << "':'";                   break;
+      case type::scope:       o << "'::'";                  break;
       case type::less:        o << "'<'";                   break;
       case type::greater:     o << "'>'";                   break;
       case type::lcbrace:     o << "'{'";                   break;
diff --git a/libbuild2/cc/lexer.hxx b/libbuild2/cc/lexer.hxx
index b4e1045..dc392c6 100644
--- a/libbuild2/cc/lexer.hxx
+++ b/libbuild2/cc/lexer.hxx
@@ -40,6 +40,8 @@ namespace build2
 
       dot,         // .
       semi,        // ;
+      colon,       // :
+      scope,       // ::
       less,        // <
       greater,     // >
       lcbrace,     // {
diff --git a/libbuild2/cc/parser+module.test.testscript b/libbuild2/cc/parser+module.test.testscript
index e4ec139..5afb0b1 100644
--- a/libbuild2/cc/parser+module.test.testscript
+++ b/libbuild2/cc/parser+module.test.testscript
@@ -7,6 +7,30 @@
 # NOTE: currently header unit imports don't produce anything.
 #
 
+: module-iface
+:
+$* <<EOI >>EOI
+export module foo;
+EOI
+
+: module-impl
+:
+$* <<EOI >>EOI
+module foo;
+EOI
+
+: module-iface-part
+:
+$* <<EOI >>EOI
+export module foo:part;
+EOI
+
+: module-impl-part
+:
+$* <<EOI >>EOI
+module foo.bar:part.sub;
+EOI
+
 : import
 :
 $* <<EOI >>EOI
@@ -23,16 +47,12 @@ import <sys/foo.h>;
 __import "/usr/include/stdio.h";
 EOI
 
-: module-implementation
+: import-part
 :
 $* <<EOI >>EOI
 module foo;
-EOI
-
-: module-interface
-:
-$* <<EOI >>EOI
-export module foo;
+import :part;
+import :part.sub;
 EOI
 
 : export-imported
@@ -54,6 +74,7 @@ export import(*a);
 import::inner xi = {};
 ::import <a>;
 class import<int>;
+import ::x;
 EOI
 
 : non-module
@@ -72,12 +93,12 @@ EOI
 : attribute
 :
 $* <<EOI >>EOO
+module bar [[module({module})]];
 import foo [[export({import})]];
 import "foo.h" [[export({import})]];
-module bar [[module({module})]];
 EOI
-import foo;
 module bar;
+import foo;
 EOO
 
 : import-duplicate
@@ -92,6 +113,16 @@ import foo;
 import bar.baz;
 EOO
 
+: part-out-purview
+:
+$* <<EOI 2>>EOE != 0
+module;
+import :part
+module foo;
+EOI
+<stdin>:2:8: error: partition importation out of module purview
+EOE
+
 : brace-missing
 :
 $* <<EOI 2>>EOE
@@ -125,7 +156,7 @@ EOE
 $* <<EOI 2>>EOE != 0
 module ;
 EOI
-<stdin>:1:1: error: module declaration expected after leading module marker
+<stdin>:1:1: error: module declaration expected after global module fragment
 EOE
 
 : import-missing-semi
diff --git a/libbuild2/cc/parser.cxx b/libbuild2/cc/parser.cxx
index 8f97fcc..976fd89 100644
--- a/libbuild2/cc/parser.cxx
+++ b/libbuild2/cc/parser.cxx
@@ -66,10 +66,11 @@ namespace build2
           {
             // Constructs we need to recognize:
             //
-            //           module                              ;
-            // [export]  module <module-name> [<attributes>] ;
-            // [export]  import <module-name> [<attributes>] ;
-            // [export]  import <header-name> [<attributes>] ;
+            //           module                                              ;
+            // [export]  module <module-name> [<module-part>] [<attributes>] ;
+            // [export]  import <module-name>                 [<attributes>] ;
+            // [export]  import <module-part>                 [<attributes>] ;
+            // [export]  import <header-name>                 [<attributes>] ;
             //
             // The leading module/export/import keyword should be the first
             // token of a logical line and only if certain characters appear
@@ -117,6 +118,7 @@ namespace build2
                 l_->next (t);
 
                 if ((t.type == type::less     ||
+                     t.type == type::colon    ||
                      t.type == type::string   ||
                      t.type == type::identifier) && !t.first)
                   parse_import (t, ex);
@@ -153,13 +155,52 @@ namespace build2
 
       if (module_marker_ && u.module_info.name.empty ())
         fail (*module_marker_) << "module declaration expected after "
-                               << "leading module marker";
+                               << "global module fragment";
 
       checksum = l.checksum ();
       return u;
     }
 
     void parser::
+    parse_module (token& t, bool ex, location_value l)
+    {
+      // enter: token after module keyword (l is the module keyword location)
+      // leave: semi
+
+      // Handle the leading 'module;' marker (p0713).
+      //
+      // Note that we don't bother diagnosing invalid/duplicate markers
+      // leaving that to the compiler.
+      //
+      if (!ex && t.type == type::semi && !t.first)
+      {
+        module_marker_ = move (l);
+        return;
+      }
+
+      // Otherwise it should be the start of the module name.
+      //
+      string n (parse_module_name (t, true /* partition */));
+
+      // Should be {}-balanced.
+      //
+      for (;
+           t.type != type::eos && t.type != type::semi && !t.first;
+           l_->next (t)) ;
+
+      if (t.type != type::semi)
+        fail (t) << "';' expected instead of " << t;
+      else if (t.first)
+        fail (t) << "';' must be on the same line";
+
+      if (!u_->module_info.name.empty ())
+        fail (l) << "multiple module declarations";
+
+      u_->type = ex ? unit_type::module_iface : unit_type::module_impl;
+      u_->module_info.name = move (n);
+    }
+
+    void parser::
     parse_import (token& t, bool ex)
     {
       // enter: token after import keyword
@@ -176,9 +217,25 @@ namespace build2
           ut = unit_type::module_header;
           break;
         }
+      case type::colon:
+        {
+          if (u_->type != unit_type::module_iface &&
+              u_->type != unit_type::module_impl)
+            fail (t) << "partition importation out of module purview";
+
+          un = parse_module_part (t);
+          ut = unit_type::module_iface; // @@ _part?
+          break;
+        }
       case type::identifier:
         {
-          un = parse_module_name (t);
+          // Note that in import a partition can only be specified without a
+          // module name. In other words, the following is invalid:
+          //
+          // module m;
+          // import m:p;
+          //
+          un = parse_module_name (t, false /* partition */);
           ut = unit_type::module_iface;
           break;
         }
@@ -223,59 +280,53 @@ namespace build2
         i->exported = i->exported || ex;
     }
 
-    void parser::
-    parse_module (token& t, bool ex, location_value l)
+    string parser::
+    parse_module_name (token& t, bool part)
     {
-      // enter: token after module keyword (l is the module keyword location)
-      // leave: semi
+      // enter: first token of module name
+      // leave: token after module name
 
-      // Handle the leading 'module;' marker (p0713).
-      //
-      // Note that we don't bother diagnosing invalid/duplicate markers
-      // leaving that to the compiler.
+      string n;
+
+      // <identifier>[ . <identifier>]* [<module-part>]
       //
-      if (!ex && t.type == type::semi && !t.first)
+      for (;; l_->next (t))
       {
-        module_marker_ = move (l);
-        return;
-      }
+        if (t.type != type::identifier)
+          fail (t) << "module name expected instead of " << t;
+        else if (t.first)
+          fail (t) << "module name must be on the same line";
 
-      // Otherwise it should be the start of the module name.
-      //
-      string n (parse_module_name (t));
+        n += t.value;
 
-      // Should be {}-balanced.
-      //
-      for (;
-           t.type != type::eos && t.type != type::semi && !t.first;
-           l_->next (t)) ;
+        if (l_->next (t) != type::dot || t.first)
+          break;
 
-      if (t.type != type::semi)
-        fail (t) << "';' expected instead of " << t;
-      else if (t.first)
-        fail (t) << "';' must be on the same line";
+        n += '.';
+      }
 
-      if (!u_->module_info.name.empty ())
-        fail (l) << "multiple module declarations";
+      if (part && t.type == type::colon && !t.first)
+        parse_module_part (t, n);
 
-      u_->type = ex ? unit_type::module_iface : unit_type::module_impl;
-      u_->module_info.name = move (n);
+      return n;
     }
 
-    string parser::
-    parse_module_name (token& t)
+    void parser::
+    parse_module_part (token& t, string& n)
     {
-      // enter: first token of module name
-      // leave: token after module name
+      // enter: colon
+      // leave: token after module partition
 
-      string n;
+      n += ':';
 
-      // <identifier>[ . <identifier>]*
+      // : <identifier>[ . <identifier>]*
       //
-      for (;; l_->next (t))
+      for (;;)
       {
-        if (t.type != type::identifier || t.first)
-          fail (t) << "module name expected instead of " << t;
+        if (l_->next (t) != type::identifier)
+          fail (t) << "partition name expected instead of " << t;
+        else if (t.first)
+          fail (t) << "partition name must be on the same line";
 
         n += t.value;
 
@@ -284,8 +335,6 @@ namespace build2
 
         n += '.';
       }
-
-      return n;
     }
 
     string parser::
diff --git a/libbuild2/cc/parser.hxx b/libbuild2/cc/parser.hxx
index 7c893b5..db5858e 100644
--- a/libbuild2/cc/parser.hxx
+++ b/libbuild2/cc/parser.hxx
@@ -28,13 +28,24 @@ namespace build2
 
     private:
       void
-      parse_import (token&, bool);
+      parse_module (token&, bool, location_value);
 
       void
-      parse_module (token&, bool, location_value);
+      parse_import (token&, bool);
 
       string
-      parse_module_name (token&);
+      parse_module_name (token&, bool);
+
+      string
+      parse_module_part (token& t)
+      {
+        string n;
+        parse_module_part (t, n);
+        return n;
+      }
+
+      void
+      parse_module_part (token&, string&);
 
       string
       parse_header_name (token&);
diff --git a/libbuild2/cc/parser.test.cxx b/libbuild2/cc/parser.test.cxx
index e5b3f6a..0c023d3 100644
--- a/libbuild2/cc/parser.test.cxx
+++ b/libbuild2/cc/parser.test.cxx
@@ -45,13 +45,13 @@ namespace build2
         unit u (p.parse (is, in));
         unit_type ut (u.type);
 
-        for (const module_import& m: u.module_info.imports)
-          cout << (m.exported ? "export " : "")
-               << "import " << m.name << ';' << endl;
-
         if (ut == unit_type::module_iface || ut == unit_type::module_impl)
           cout << (ut == unit_type::module_iface ? "export " : "")
                << "module " << u.module_info.name << ';' << endl;
+
+        for (const module_import& m: u.module_info.imports)
+          cout << (m.exported ? "export " : "")
+               << "import " << m.name << ';' << endl;
       }
       catch (const failed&)
       {