Replace docquery to note-watcher, which supports lunr search index loading

NoteWatcher holds NoteCache, which caches recent notes and file modified time to help search index recovery.

lib/notational-velocity-view.coffee

@@ -3,14 +3,15 @@ fs = require 'fs-plus'
 _ = require 'underscore-plus'
 {$, $$, SelectListView} = require 'atom-space-pen-views'
 DocQuery = require 'docquery'
+NoteWatcher = require './note-watcher'
 Utility = require './utility'
 
 module.exports =
 class NotationalVelocityView extends SelectListView
   initialize: (state) ->
-    @initializedAt = new Date()
     super
     @addClass('nvatom')
+    @maxItems = 100
     @rootDirectory = Utility.getNoteDirectory()
     unless fs.existsSync(@rootDirectory)
       throw new Error("The given directory #{@rootDirectory} does not exist. "
@@ -18,19 +19,15 @@ class NotationalVelocityView extends SelectListView
     @skipPopulateList = false
     @prevCursorPosition = 0
     @documentsLoaded = false
-    @docQuery = new DocQuery(@rootDirectory, {recursive: true, extensions: atom.config.get('nvatom.extensions')})
-    @docQuery.on "ready", () =>
+    @noteWatcher = new NoteWatcher(@rootDirectory, atom.config.get('nvatom.extensions'), @maxItems)
+    @noteWatcher.on "ready", () =>
       @documentsLoaded = true
       @setLoading()
       @populateList()
-    @docQuery.on "added", (fileDetails) =>
-      @populateList() if @documentsLoaded
-    @docQuery.on "updated", (fileDetails) =>
-      @populateList() if @documentsLoaded
-    @docQuery.on "removed", (fileDetails) =>
+    @noteWatcher.on "update", () => 
       @populateList() if @documentsLoaded
     unless atom.config.get('nvatom.enableLunrPipeline')
-      @docQuery.searchIndex.pipeline.reset()
+      @noteWatcher.searchIndex.pipeline.reset()
 
   isCursorProceeded: ->
     editor = @filterEditorView.model
@@ -62,9 +59,7 @@ class NotationalVelocityView extends SelectListView
         @selectItemView(@list.find("li:nth-child(#{n})"))
 
   filter: (filterQuery) ->
-    if (filterQuery is "") or (filterQuery is undefined)
-      return @docQuery.documents
-    return @docQuery.search(filterQuery)
+    @noteWatcher.search(filterQuery)
 
   getFilterKey: ->
     'filetext'
@@ -156,7 +151,7 @@ class NotationalVelocityView extends SelectListView
       @selectItem(filteredItems, filterQuery)
 
     else
-      @setError(@getEmptyMessage(@docQuery.documents.length, filteredItems.length))
+      @setError(@getEmptyMessage(@noteWatcher.length, filteredItems.length))
 
   schedulePopulateList: ->
     unless @skipPopulateList

lib/note-cache.coffee

@@ -0,0 +1,136 @@
+fs = require 'fs-plus'
+path = require 'path'
+
+# Keeps a track of notes and its modified time.
+#
+# It caches recent notes. Note that a note may partially contain its file content. A file system watcher should call
+# `upsert` and `remove` accordingly once the note cache is created.
+#
+# TODO: Is it a good design? On creation, it deals with file system directly. Once it is created, it relies on outer
+#       feedback, instead of monitoring the file system by itself.
+#
+module.exports =
+class NoteCache
+  constructor: (@_baseDirectory, @_maxItem, @_maxNoteLength) ->
+    @_maxItem = @_maxItem ? 100
+    @_maxNoteLength = @_maxNoteLength ? 100
+    @_noteStats = {}
+    @_state = 'init'
+  
+  load: (noteStats) ->
+    @_noteStats = noteStats
+    @_state = 'cache'
+    this
+  
+  ready: ->
+    @_buildNoteSortedList()
+    @_buildNoteCache()
+    @_assert()
+    @_state = 'ready'
+    this
+      
+  toJSON: ->
+    @_assertReady()
+    JSON.stringify(@_noteStats)
+
+  upsert: (noteId, mtime) ->
+    @_noteStats[noteId] = mtime.getTime()
+    return this unless @_state == 'ready'
+    
+    if !(noteId in @_noteSortedList)
+      @_noteSortedList.push(noteId)
+    # TODO: Can this be improved?
+    @_noteSortedList.sort(@_noteIdCompare)
+    if @_noteSortedList.indexOf(noteId) < @_maxItem
+      @_noteCache[noteId] = @_buildNote(noteId)
+      if @_noteSortedList.length > @_maxItem and @_noteSortedList[@_maxItem] in Object.keys(@_noteCache)
+        delete @_noteCache[@_noteSortedList[@_maxItem]]
+    @_assert()
+    this
+  
+  remove: (noteId) ->
+    delete @_noteStats[noteId]
+    return this unless @_state == 'ready'
+    
+    if noteId in Object.keys(@_noteCache)
+      delete @_noteCache[noteId]
+      if @_noteSortedList.length > @_maxItem and !(@_noteSortedList[@_maxItem] in Object.keys(@_noteCache))
+        @_noteCache[@_noteSortedList[@_maxItem]] = @_buildNote(@_noteSortedList[@_maxItem])
+    @_noteSortedList.splice(@_noteSortedList.indexOf(noteId), 1)
+    @_assert()
+    this
+
+  getNote: (noteId) ->
+    @_assertReady()
+    if noteId in @_noteCache then @_noteCache[noteId] else @_buildNote(noteId)
+  
+  getRecentNotes: ->
+    @_assertReady()
+    @_noteSortedList
+      .slice(0, @_maxItem)
+      .map((noteId) => @_noteCache[noteId])
+  
+  hasNoteId: (noteId) ->
+    @_noteStats.hasOwnProperty(noteId)
+    
+  getNoteIds: ->
+    @_noteSortedList ? Object.keys(@_noteStats)
+  
+  getModifiedAt: (noteId) ->
+    @_noteStats[noteId]
+
+  length: -> 
+    @_assertReady()
+    @_noteSortedList.length
+  
+  _assertReady: ->
+    throw new Error "state is not ready; #{@_state}" unless @_state == 'ready'
+  
+  _assert: ->
+    throw new Error "cache length is wrong; #{Object.keys(@_noteCache).length} #{@_noteSortedList} #{@_maxItem}" unless Object.keys(@_noteCache).length == Math.min(@_maxItem, @_noteSortedList.length)
+    throw new Error 'list length is wrong' unless @_noteSortedList.length == Object.keys(@_noteStats).length
+
+  # _buildNoteStats: ->
+  #   ret = {}
+  #   visited = [fs.absolute(@_baseDirectory)]
+  #   fs.traverseTreeSync(
+  #     @_baseDirectory,
+  #     (filePath) =>
+  #       fileName = path.basename(filePath)
+  #       if @_extensions.indexOf(path.extname(fileName)) >= 0
+  #         noteId = path.relative(@_baseDirectory, filePath)
+  #         ret[noteId] = fs.statSync(filePath).mtime.getTime()
+  #     ,
+  #     (directoryPath) =>
+  #       return false if fs.absolute(directoryPath) in visited
+  #       visited.push(fs.absolute(directoryPath))
+  #       return true
+  #   )
+  #   ret
+  
+  _buildNoteSortedList: ->
+    @_noteSortedList = Object.keys(@_noteStats)
+    @_noteSortedList.sort(@_noteIdCompare)
+    
+  _buildNoteCache: ->
+    @_noteCache = {}
+    for noteId in @_noteSortedList.slice(0, @_maxItem)
+      @_noteCache[noteId] = @_buildNote(noteId)
+    
+  _buildNote: (noteId) ->
+    filePath = path.join(@_baseDirectory, noteId)
+    fileName = path.basename(filePath)
+    # TODO: Verify it is right
+    title = path.basename(fileName, path.extname(fileName))
+    body = ''
+    if fs.existsSync(filePath)
+      # TODO: Read just the first n letters.
+      body = fs.readFileSync(filePath, 'utf8').slice(0, @_maxNoteLength)
+    return { title: title, body: body, filePath: filePath, modifiedAt: fs.statSync(filePath).mtime }
+
+  _noteIdCompare: (a, b) =>
+    if @_noteStats[a] > @_noteStats[b]
+      return -1
+    if @_noteStats[a] < @_noteStats[b]
+      return 1
+    return 0

lib/note-watcher.coffee

@@ -0,0 +1,130 @@
+chokidar = require 'chokidar'
+fs = require 'fs'
+lunr = require 'lunr'
+path = require 'path'
+_ = require 'underscore-plus'
+zlib = require 'zlib'
+{EventEmitter} = require 'events'
+NoteCache = require './note-cache'
+
+module.exports =
+class NoteWatcher extends EventEmitter
+  @cacheVersion = 0
+  
+  constructor: (@_baseDirectory, @_extensions, @_maxItems) ->
+    @_maxItems = @_maxItems ? 100
+    @_extensions = @_extensions ? ['.txt', '.md']
+    @_noteCache = new NoteCache(@_baseDirectory, @_maxItems)
+    @_restoreSearchIndex() || @_initSearchIndex()
+    @_startWatcher()
+
+  save: ->
+    cache = {
+      baseDirectory: @_baseDirectory,
+      extensions: @_extensions,
+      version: @_cacheVersion,
+      searchIndex: JSON.stringify(@_searchIndex),
+      noteCache: @_noteCache.toJSON(),
+    }
+    fs.writeFileSync(
+      path.join(@_baseDirectory, 'nvatom.cache'), 
+      zlib.deflateSync(JSON.stringify(cache)))
+  
+  close: ->
+    @_watcher.close()
+  
+  search: (query) ->
+    if query? and query.length > 0
+      return @_searchIndex.search(query)
+        .slice(0, @_maxItems)
+        .map((x) => @_noteCache.getNote(x.ref))
+    else
+      return @_noteCache.getRecentNotes()
+  
+  length: -> 
+    @_noteCache.length()
+    
+  _initSearchIndex: ->
+    @_state = 'initializing'
+    @_searchIndex = lunr(() ->
+      @field('title', { boost: 10 })
+      @field('body')
+    )
+  
+  _restoreSearchIndex: ->
+    cacheFilePath = path.join(@_baseDirectory, 'nvatom.cache')
+    return false unless fs.existsSync(cacheFilePath)
+    cache = JSON.parse(zlib.inflateSync(fs.readFileSync(cacheFilePath)))
+    return false if cache == null
+    return false unless _.isEqual(cache.baseDirectory, @_baseDirectory)
+    return false unless _.isEqual(cache.extensions, @_extensions)
+    return false unless _.isEqual(cache.version, @_cacheVersion)
+    @_searchIndex = lunr.Index.load(JSON.parse(cache.searchIndex))
+    # TODO: Make this as an interface. Old note cache does not have to know upsert, remove, ready and so on.
+    #       It only needs to know getNoteIds, hasNoteId, and getModifiedAt.
+    @_oldNoteCache = new NoteCache(@_baseDirectory, @_maxItems).load(JSON.parse(cache.noteCache))
+    @_state = 'recovering'
+    return true
+
+  _startWatcher: ->
+    options = {
+      ignored: (filePath, fileStat) =>
+        if fileStat?.isFile() then @_extensions.indexOf(path.extname(filePath)) < 0 else false
+    }
+    
+    @_watcher = chokidar
+      .watch @_baseDirectory, options
+      .on 'add', (args) => @_add(args)
+      .on 'change', (args) => @_change(args)
+      .on 'unlink', (args) => @_unlink(args)
+      .on 'ready', () => @_ready()
+
+  _add: (filePath) ->
+    @_noteCache.upsert(@_toNoteId(filePath), fs.statSync(filePath).mtime)
+    if @_state != 'recovering'
+      @_searchIndex.add(@_toFileDetails(filePath))
+    @emit 'update', filePath
+  
+  _change: (filePath) ->
+    @_noteCache.upsert(@_toNoteId(filePath), fs.statSync(filePath).mtime)
+    if @_state != 'recovering'
+      @_searchIndex.update(@_toFileDetails(filePath))
+    @emit 'update', filePath
+  
+  _unlink: (filePath) ->
+    @_noteCache.remove(@_toNoteId(filePath))
+    if @_state != 'recovering'
+      @_searchIndex.remove(@_toFileDetails(filePath))
+    @emit 'update', filePath
+  
+  _ready: () -> 
+    @_noteCache.ready()
+    if @_state == 'recovering'
+      @_updateSearchIndex(@_oldNoteCache, @_noteCache)
+      delete @_oldNoteCache
+    @_state = 'ready'
+    @emit 'ready'
+  
+  _updateSearchIndex: (oldNoteCache, newNoteCache) ->
+    for noteId in oldNoteCache.getNoteIds()
+      if !newNoteCache.hasNoteId(noteId)
+        @_searchIndex.remove(@_toFileDetails(@_toFilePath(noteId)))
+      else if oldNoteCache.getModifiedAt(noteId) <= newNoteCache.getModifiedAt(noteId)
+        @_searchIndex.update(@_toFileDetails(@_toFilePath(noteId)))
+    for noteId in newNoteCache.getNoteIds()
+      if !oldNoteCache.hasNoteId(noteId)
+        @_searchIndex.add(@_toFileDetails(@_toFilePath(noteId)))
+  
+  _toNoteId: (filePath) ->
+    path.relative(@_baseDirectory, filePath)
+  
+  _toFilePath: (noteId) ->
+    path.join(@_baseDirectory, noteId)
+  
+  _toFileDetails: (filePath) ->
+    fileName = path.basename(filePath)
+    return {
+      id: @_toNoteId(filePath),
+      title: path.basename(fileName, path.extname(fileName)),
+      body: if fs.existsSync(filePath) then fs.readFileSync(filePath, { encoding: 'utf8' }) else ''
+    }