[Gallery] Fix directory structure (#312)

gallery/tool/codeviewer_cli/README.md

@@ -0,0 +1,73 @@
+# Codeviewer
+
+A command-line application to highlight dart source code.
+
+## Overview
+
+Code segments are highlighted before the app is compiled.
+This is done because the highlighting process can take 300ms to finish, creating
+a noticeable delay when the demo switches to code page.
+
+The highlighter takes all files in the `gallery/lib/demos/` folder and scans each.
+Highlighted code widgets are stored in the
+`gallery/lib/codeviewer/code_segments.dart` file.
+
+## How to generate code segments
+
+From the `samples/gallery/` directory:
+1. Make sure you have [grinder](https://pub.dev/packages/grinder) installed by
+running `flutter pub get`.
+2. Then run `flutter pub run grinder update-code-segments` to generate code
+segments with highlighting.
+
+## How to define a block of code to generate highlighting for
+
+Wrap a block of code with lines `// BEGIN yourDemoName` and `// END` to mark it
+for highlighting. The block in between, as well as any copyright notice and
+imports at the beginning of the file, are automatically taken and highlighted,
+and stored as `static TextSpan yourDemoName(BuildContext context)` in
+`gallery/lib/codeviewer/code_segments.dart`. To display the code, go to
+`gallery/lib/data/demos.dart`, and add `code: CodeSegments.yourDemoName,` to
+your `GalleryDemoConfiguration` object.
+
+## Multiple blocks of code
+
+Use the following method to join multiple blocks of code into a single segment:
+```
+// BEGIN yourDemo#2
+a();
+// END
+b();
+// BEGIN yourDemo#1
+c();
+// END
+```
+The generated code will be
+```
+c();
+a();
+```
+
+Code blocks can nest or overlap. In these cases, specify which file(s) to `END`.
+
+The following source file
+```
+// BEGIN demoOne
+a();
+// BEGIN demoTwo
+b();
+// END demoOne
+c();
+// END demoTwo
+```
+will create the following segments:
+(demoOne)
+```
+a();
+b();
+```
+(demoTwo)
+```
+b();
+c();
+```

gallery/tool/codeviewer_cli/main.dart

@@ -0,0 +1,31 @@
+// Copyright 2019 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:path/path.dart' as path;
+
+import 'segment_generator.dart';
+
+void main(List<String> arguments) {
+  final parser = ArgParser()
+    ..addOption(
+      'target',
+      help: 'The file path for the output target file.',
+      defaultsTo: path.join(
+          Directory.current.path, 'lib', 'codeviewer', 'code_segments.dart'),
+    )
+    ..addFlag(
+      'dry-run',
+      help: 'Write the output to stdout.',
+    );
+  final argResults = parser.parse(arguments);
+
+  writeSegments(
+    sourceDirectoryPath: path.join(Directory.current.path, 'lib', 'demos'),
+    targetFilePath: argResults['target'] as String,
+    isDryRun: argResults['dry-run'] as bool,
+  );
+}

gallery/tool/codeviewer_cli/prehighlighter.dart

@@ -0,0 +1,412 @@
+// Copyright 2019 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:string_scanner/string_scanner.dart';
+
+abstract class SyntaxPrehighlighter {
+  List<CodeSpan> format(String src);
+}
+
+class DartSyntaxPrehighlighter extends SyntaxPrehighlighter {
+  DartSyntaxPrehighlighter() {
+    _spans = <_HighlightSpan>[];
+  }
+
+  static const List<String> _keywords = <String>[
+    'abstract',
+    'as',
+    'assert',
+    'async',
+    'await',
+    'break',
+    'case',
+    'catch',
+    'class',
+    'const',
+    'continue',
+    'default',
+    'deferred',
+    'do',
+    'dynamic',
+    'else',
+    'enum',
+    'export',
+    'external',
+    'extends',
+    'factory',
+    'false',
+    'final',
+    'finally',
+    'for',
+    'get',
+    'if',
+    'implements',
+    'import',
+    'in',
+    'is',
+    'library',
+    'new',
+    'null',
+    'operator',
+    'part',
+    'rethrow',
+    'return',
+    'set',
+    'static',
+    'super',
+    'switch',
+    'sync',
+    'this',
+    'throw',
+    'true',
+    'try',
+    'typedef',
+    'var',
+    'void',
+    'while',
+    'with',
+    'yield',
+  ];
+
+  static const List<String> _builtInTypes = <String>[
+    'int',
+    'double',
+    'num',
+    'bool',
+  ];
+
+  String _src;
+  StringScanner _scanner;
+
+  List<_HighlightSpan> _spans;
+
+  @override
+  List<CodeSpan> format(String src) {
+    _src = src;
+    _scanner = StringScanner(_src);
+
+    if (_generateSpans()) {
+      // Successfully parsed the code
+      final List<CodeSpan> formattedText = <CodeSpan>[];
+      int currentPosition = 0;
+
+      for (_HighlightSpan span in _spans) {
+        if (currentPosition != span.start) {
+          formattedText
+              .add(CodeSpan(text: _src.substring(currentPosition, span.start)));
+        }
+
+        formattedText
+            .add(CodeSpan(type: span.type, text: span.textForSpan(_src)));
+
+        currentPosition = span.end;
+      }
+
+      if (currentPosition != _src.length) {
+        formattedText
+            .add(CodeSpan(text: _src.substring(currentPosition, _src.length)));
+      }
+
+      return formattedText;
+    } else {
+      // Parsing failed, return with only basic formatting
+      return [CodeSpan(type: _HighlightType.base, text: src)];
+    }
+  }
+
+  bool _generateSpans() {
+    int lastLoopPosition = _scanner.position;
+
+    while (!_scanner.isDone) {
+      // Skip White space
+      _scanner.scan(RegExp(r'\s+'));
+
+      // Block comments
+      if (_scanner.scan(RegExp(r'/\*(.|\n)*\*/'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.comment,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Line comments
+      if (_scanner.scan('//')) {
+        final int startComment = _scanner.lastMatch.start;
+
+        bool eof = false;
+        int endComment;
+        if (_scanner.scan(RegExp(r'.*\n'))) {
+          endComment = _scanner.lastMatch.end - 1;
+        } else {
+          eof = true;
+          endComment = _src.length;
+        }
+
+        _spans.add(_HighlightSpan(
+          _HighlightType.comment,
+          startComment,
+          endComment,
+        ));
+
+        if (eof) {
+          break;
+        }
+
+        continue;
+      }
+
+      // Raw r"String"
+      if (_scanner.scan(RegExp(r'r".*"'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.string,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Raw r'String'
+      if (_scanner.scan(RegExp(r"r'.*'"))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.string,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Multiline """String"""
+      if (_scanner.scan(RegExp(r'"""(?:[^"\\]|\\(.|\n))*"""'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.string,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Multiline '''String'''
+      if (_scanner.scan(RegExp(r"'''(?:[^'\\]|\\(.|\n))*'''"))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.string,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // "String"
+      if (_scanner.scan(RegExp(r'"(?:[^"\\]|\\.)*"'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.string,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // 'String'
+      if (_scanner.scan(RegExp(r"'(?:[^'\\]|\\.)*'"))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.string,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Double
+      if (_scanner.scan(RegExp(r'\d+\.\d+'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.number,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Integer
+      if (_scanner.scan(RegExp(r'\d+'))) {
+        _spans.add(_HighlightSpan(_HighlightType.number,
+            _scanner.lastMatch.start, _scanner.lastMatch.end));
+        continue;
+      }
+
+      // Punctuation
+      if (_scanner.scan(RegExp(r'[\[\]{}().!=<>&\|\?\+\-\*/%\^~;:,]'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.punctuation,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Meta data
+      if (_scanner.scan(RegExp(r'@\w+'))) {
+        _spans.add(_HighlightSpan(
+          _HighlightType.keyword,
+          _scanner.lastMatch.start,
+          _scanner.lastMatch.end,
+        ));
+        continue;
+      }
+
+      // Words
+      if (_scanner.scan(RegExp(r'\w+'))) {
+        _HighlightType type;
+
+        String word = _scanner.lastMatch[0];
+        if (word.startsWith('_')) {
+          word = word.substring(1);
+        }
+
+        if (_keywords.contains(word)) {
+          type = _HighlightType.keyword;
+        } else if (_builtInTypes.contains(word)) {
+          type = _HighlightType.keyword;
+        } else if (_firstLetterIsUpperCase(word)) {
+          type = _HighlightType.klass;
+        } else if (word.length >= 2 &&
+            word.startsWith('k') &&
+            _firstLetterIsUpperCase(word.substring(1))) {
+          type = _HighlightType.constant;
+        }
+
+        if (type != null) {
+          _spans.add(_HighlightSpan(
+            type,
+            _scanner.lastMatch.start,
+            _scanner.lastMatch.end,
+          ));
+        }
+      }
+
+      // Check if this loop did anything
+      if (lastLoopPosition == _scanner.position) {
+        // Failed to parse this file, abort gracefully
+        return false;
+      }
+      lastLoopPosition = _scanner.position;
+    }
+
+    _simplify();
+    return true;
+  }
+
+  void _simplify() {
+    for (int i = _spans.length - 2; i >= 0; i -= 1) {
+      if (_spans[i].type == _spans[i + 1].type &&
+          _spans[i].end == _spans[i + 1].start) {
+        _spans[i] = _HighlightSpan(
+          _spans[i].type,
+          _spans[i].start,
+          _spans[i + 1].end,
+        );
+        _spans.removeAt(i + 1);
+      }
+    }
+  }
+
+  bool _firstLetterIsUpperCase(String str) {
+    if (str.isNotEmpty) {
+      final String first = str.substring(0, 1);
+      return first == first.toUpperCase();
+    }
+    return false;
+  }
+}
+
+enum _HighlightType {
+  number,
+  comment,
+  keyword,
+  string,
+  punctuation,
+  klass,
+  constant,
+  base,
+}
+
+class _HighlightSpan {
+  _HighlightSpan(this.type, this.start, this.end);
+  final _HighlightType type;
+  final int start;
+  final int end;
+
+  String textForSpan(String src) {
+    return src.substring(start, end);
+  }
+}
+
+class CodeSpan {
+  CodeSpan({this.type = _HighlightType.base, this.text});
+
+  final _HighlightType type;
+  final String text;
+
+  @override
+  String toString() {
+    return 'TextSpan('
+        'style: codeStyle.${_styleNameOf(type)}, '
+        "text: '${_escape(text)}'"
+        ')';
+  }
+}
+
+String _styleNameOf(_HighlightType type) {
+  switch (type) {
+    case _HighlightType.number:
+      return 'numberStyle';
+    case _HighlightType.comment:
+      return 'commentStyle';
+    case _HighlightType.keyword:
+      return 'keywordStyle';
+    case _HighlightType.string:
+      return 'stringStyle';
+    case _HighlightType.punctuation:
+      return 'punctuationStyle';
+    case _HighlightType.klass:
+      return 'classStyle';
+    case _HighlightType.constant:
+      return 'constantStyle';
+    case _HighlightType.base:
+      return 'baseStyle';
+  }
+  return '';
+}
+
+String _escape(String text) {
+  StringBuffer escapedText = StringBuffer();
+
+  for (final char in text.runes) {
+    if (char < 0x20 ||
+        char >= 0x7F ||
+        char == 0x22 ||
+        char == 0x24 ||
+        char == 0x27 ||
+        char == 0x5C) {
+      if (char <= 0xffff) {
+        escapedText.write("\\u${_encodeAndPad(char)}");
+      } else {
+        escapedText.write("\\u{${_encode(char)}}");
+      }
+    } else {
+      escapedText.write(String.fromCharCode(char));
+    }
+  }
+
+  return escapedText.toString();
+}
+
+String _encode(int charCode) {
+  return charCode.toRadixString(16);
+}
+
+String _encodeAndPad(int charCode) {
+  String encoded = _encode(charCode);
+  return '0' * (4 - encoded.length) + encoded;
+}

gallery/tool/codeviewer_cli/segment_generator.dart

@@ -0,0 +1,267 @@
+// Copyright 2019 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+import 'prehighlighter.dart';
+
+const _globalPrologue =
+    '''// This file is automatically generated by codeviewer_cli.
+// Do not edit this file.
+
+// Copyright 2019 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter/material.dart';
+import 'package:gallery/codeviewer/code_style.dart';
+
+class CodeSegments {
+''';
+
+const _globalEpilogue = '}\n';
+
+final Pattern beginSubsegment = RegExp(r'//\s+BEGIN');
+final Pattern endSubsegment = RegExp(r'//\s+END');
+
+enum _FileReadStatus {
+  comments,
+  imports,
+  finished,
+}
+
+/// Returns the new status of the scanner whose previous status was
+/// [oldStatus], after scanning the line [line].
+_FileReadStatus _updatedStatus(_FileReadStatus oldStatus, String line) {
+  _FileReadStatus lineStatus;
+  if (line.trim().startsWith('//')) {
+    lineStatus = _FileReadStatus.comments;
+  } else if (line.trim().startsWith('import')) {
+    lineStatus = _FileReadStatus.imports;
+  } else {
+    lineStatus = _FileReadStatus.finished;
+  }
+
+  _FileReadStatus newStatus;
+  switch (oldStatus) {
+    case _FileReadStatus.comments:
+      newStatus =
+          (line.trim().isEmpty || lineStatus == _FileReadStatus.comments)
+              ? _FileReadStatus.comments
+              : lineStatus;
+      break;
+    case _FileReadStatus.imports:
+      newStatus = (line.trim().isEmpty || lineStatus == _FileReadStatus.imports)
+          ? _FileReadStatus.imports
+          : _FileReadStatus.finished;
+      break;
+    case _FileReadStatus.finished:
+      newStatus = oldStatus;
+      break;
+  }
+  return newStatus;
+}
+
+Map<String, String> _createSegments(String sourceDirectoryPath) {
+  List<File> files = Directory(sourceDirectoryPath)
+      .listSync(recursive: true)
+      .whereType<File>()
+      .toList();
+
+  Map<String, StringBuffer> subsegments = {};
+  Map<String, String> subsegmentPrologues = {};
+
+  Set<String> appearedSubsegments = Set();
+
+  for (final file in files) {
+    // Process file.
+
+    String content = file.readAsStringSync();
+    List<String> lines = LineSplitter().convert(content);
+
+    _FileReadStatus status = _FileReadStatus.comments;
+
+    StringBuffer prologue = StringBuffer();
+
+    Set<String> activeSubsegments = Set();
+
+    for (final line in lines) {
+      // Update status.
+
+      status = _updatedStatus(status, line);
+
+      if (status != _FileReadStatus.finished) {
+        prologue.writeln(line);
+      }
+
+      // Process run commands.
+
+      if (line.trim().startsWith(beginSubsegment)) {
+        String argumentString = line.replaceFirst(beginSubsegment, '').trim();
+        List<String> arguments =
+            argumentString.isEmpty ? [] : argumentString.split(RegExp(r'\s+'));
+
+        for (final argument in arguments) {
+          if (activeSubsegments.contains(argument)) {
+            throw PreformatterException(
+                'BEGIN $argument is used twice in file ${file.path}');
+          } else if (appearedSubsegments.contains(argument)) {
+            throw PreformatterException('BEGIN $argument is used twice');
+          } else {
+            activeSubsegments.add(argument);
+            appearedSubsegments.add(argument);
+            subsegments[argument] = StringBuffer();
+            subsegmentPrologues[argument] = prologue.toString();
+          }
+        }
+      } else if (line.trim().startsWith(endSubsegment)) {
+        String argumentString = line.replaceFirst(endSubsegment, '').trim();
+        List<String> arguments =
+            argumentString.isEmpty ? [] : argumentString.split(RegExp(r'\s+'));
+
+        if (arguments.isEmpty && activeSubsegments.length == 1) {
+          arguments.add(activeSubsegments.first);
+        }
+
+        for (final argument in arguments) {
+          if (activeSubsegments.contains(argument)) {
+            activeSubsegments.remove(argument);
+          } else {
+            throw PreformatterException(
+                'END $argument is used without a paired BEGIN in ${file.path}');
+          }
+        }
+      } else {
+        // Simple line.
+
+        for (final name in activeSubsegments) {
+          subsegments[name].writeln(line);
+        }
+      }
+    }
+
+    if (activeSubsegments.isNotEmpty) {
+      throw PreformatterException('File ${file.path} has unpaired BEGIN');
+    }
+  }
+
+  Map<String, List<TaggedString>> segments = {};
+  Map<String, String> segmentPrologues = {};
+
+  // Sometimes a code segment is made up of subsegments. They are marked by
+  // names with a "#" symbol in it, such as "bottomSheetDemoModal#1" and
+  // "bottomSheetDemoModal#2".
+  // The following code groups the subsegments by order into segments.
+  subsegments.forEach((key, value) {
+    String name;
+    double order;
+
+    if (key.contains('#')) {
+      List<String> parts = key.split('#');
+      name = parts[0];
+      order = double.parse(parts[1]);
+    } else {
+      name = key;
+      order = 0;
+    }
+
+    if (!segments.containsKey(name)) {
+      segments[name] = [];
+    }
+    segments[name].add(
+      TaggedString(
+        text: value.toString(),
+        order: order,
+      ),
+    );
+
+    segmentPrologues[name] = subsegmentPrologues[key];
+  });
+
+  segments.forEach((key, value) {
+    value.sort((ts1, ts2) => (ts1.order - ts2.order).sign.round());
+  });
+
+  Map<String, String> answer = {};
+
+  for (final name in segments.keys) {
+    StringBuffer buffer = StringBuffer();
+
+    buffer.write(segmentPrologues[name].trim());
+    buffer.write('\n\n');
+
+    for (final ts in segments[name]) {
+      buffer.write(ts.text.trim());
+      buffer.write('\n\n');
+    }
+
+    answer[name] = buffer.toString();
+  }
+
+  return answer;
+}
+
+/// A string [text] together with a number [order], for sorting purposes.
+/// Used to store different subsegments of a code segment.
+/// The [order] of each subsegment is tagged with the code in order to be
+/// sorted in the desired order.
+class TaggedString {
+  TaggedString({this.text, this.order});
+
+  final String text;
+  final double order;
+}
+
+void _formatSegments(Map<String, String> segments, IOSink output) {
+  output.write(_globalPrologue);
+
+  final sortedNames = segments.keys.toList()..sort();
+  for (final name in sortedNames) {
+    String code = segments[name];
+
+    output.writeln('  static TextSpan $name (BuildContext context) {');
+    output.writeln('    final CodeStyle codeStyle = CodeStyle.of(context);');
+    output.writeln('    return TextSpan(children: [');
+
+    List<CodeSpan> codeSpans = DartSyntaxPrehighlighter().format(code);
+
+    for (final span in codeSpans) {
+      output.write('    ');
+      output.write(span.toString());
+      output.write(',\n');
+    }
+
+    output.write('  ]); }\n');
+  }
+
+  output.write(_globalEpilogue);
+
+  output.close();
+}
+
+/// Collect code segments, highlight, and write to file.
+///
+/// [writeSegments] walks through the directory specified by
+/// [sourceDirectoryPath] and reads every file in it,
+/// collects code segments marked by "// BEGIN <segment_name>" and "// END",
+/// highlights them, and writes to the file specified by
+/// [targetFilePath]. If [isDryRun] is true, the output will
+/// be written to stdout.
+///
+/// The output file is a dart source file with a class "CodeSegments" and
+/// static methods of type TextSpan(BuildContext context).
+/// Each method generates a widget that displays a segment of code.
+///
+/// The target file is overwritten.
+void writeSegments(
+    {String sourceDirectoryPath, String targetFilePath, bool isDryRun}) {
+  Map<String, String> segments = _createSegments(sourceDirectoryPath);
+  IOSink output = isDryRun ? stdout : File(targetFilePath).openWrite();
+  _formatSegments(segments, output);
+}
+
+class PreformatterException implements Exception {
+  PreformatterException(this.cause);
+  String cause;
+}