[pylucene-dev] ***SPAM*** My awful performance experiment of PyLucene vs Lucene

[Liang Xing]

Skipped content of type multipart/alternative-------------- next part --------------
#!/usr/bin/env python

import sys, os, PyLucene, threading, time
from datetime import datetime

"""
This class is loosely based on the Lucene (java implementation) demo class 
org.apache.lucene.demo.IndexFiles.  It will take a directory as an argument
and will index all of the files in that directory and downward recursively.
It will index on the file path, the file name and the file contents.  The
resulting Lucene index will be placed in the current directory and called
'index'.
"""

class Ticker(object):

    def __init__(self):
        self.tick = True

    def run(self):
        while self.tick:
            sys.stdout.write('.')
            sys.stdout.flush()
            time.sleep(1.0)

class IndexFiles(object):
    """Usage: python IndexFiles <doc_directory>"""

    def __init__(self, root, storeDir, analyzer):

        if not os.path.exists(storeDir):
            os.mkdir(storeDir)
        store = PyLucene.FSDirectory.getDirectory(storeDir, True)
        writer = PyLucene.IndexWriter(store, analyzer, True)
        writer.setMaxFieldLength(1048576)
        self.indexDocs(root, writer)
        ticker = Ticker()
        print 'optimizing index',
        threading.Thread(target=ticker.run).start()
        writer.optimize()
        writer.close()
        ticker.tick = False
        print 'done'

    def indexDocs(self, root, writer):
        for root, dirnames, filenames in os.walk(root):
            for filename in filenames:
                if not filename.endswith('.txt'):
                    continue
                print "adding", filename
                try:
                    path = os.path.join(root, filename)
                    file = open(path)
                    contents = unicode(file.read(), 'iso-8859-1')
                    file.close()
                    doc = PyLucene.Document()
                    doc.add(PyLucene.Field("name", filename,
                                           PyLucene.Field.Store.YES,
                                           PyLucene.Field.Index.UN_TOKENIZED))
                    doc.add(PyLucene.Field("path", path,
                                           PyLucene.Field.Store.YES,
                                           PyLucene.Field.Index.UN_TOKENIZED))
                    if len(contents) > 0:
                        doc.add(PyLucene.Field("contents", contents,
                                               PyLucene.Field.Store.NO,
                                               PyLucene.Field.Index.TOKENIZED))
                    else:
                        print "warning: no content in %s" % filename
                    writer.addDocument(doc)
                except Exception, e:
                    print "Failed in indexDocs:", e

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print IndexFiles.__doc__
        sys.exit(1)
    print 'PyLucene', PyLucene.VERSION, 'Lucene', PyLucene.LUCENE_VERSION
    start = datetime.now()
    try:
        IndexFiles(sys.argv[1], "index", PyLucene.StandardAnalyzer())
        end = datetime.now()
        print end - start
    except Exception, e:
        print "Failed: ", e
-------------- next part --------------
#!/usr/bin/env python
import sys, os, threading, time
from PyLucene import QueryParser, IndexSearcher, StandardAnalyzer, FSDirectory
from PyLucene import VERSION, LUCENE_VERSION
from datetime import datetime

"""
This script is loosely based on the Lucene (java implementation) demo class 
org.apache.lucene.demo.SearchFiles.  It will prompt for a search query, then it
will search the Lucene index in the current directory called 'index' for the
search query entered against the 'contents' field.  It will then display the
'path' and 'name' fields for each of the hits it finds in the index.  Note that
search.close() is currently commented out because it causes a stack overflow in
some cases.

***Revision by Liang.xing 0.1

Revised for reading testcases from specific file, calculating the sum of time costage in searching. Additional, datetime(a standard library object) is appended.

1.UserInterface is used for interacting with user, which simply produce a name-list of files.
2.UserWord is used for fetching form a specific file, which simply produce a list of search keywords.
3.RunSearch is used for Submit the searching request, particularly, a query. It propose the search result, update the time_costing globally.

***Revision by Liang.xing 0.2

Revised for reading lines operation: merely by "for raw_inputline in fhp.readlines", which could simplify the process of reading words

"""
global time_costing, cur_idx, total_idx
class Ticker(object):
    def __init__(self):
		self.tick = True

    def run(self):
		while self.tick:
			sys.stdout.write("\r%4d/%d done(%5dms)" % (cur_idx, total_idx, time_costing))
			sys.stdout.flush()
			time.sleep(0.5)

class UserCases(object):

	def __init__(self, uParam):
		self.initflag = 1
		self.wordlist = []
		self.filelist = []
		self.filelist = uParam
		
		for file in self.filelist:
			t = self.gettitles(file)
			self.wordlist.extend(t)
			
		self.initflag = 0

	def getfilelist(self, target):
		if self.initflag:
			return self.filelist

		self.filelist.append(target)
		return self.filelist

	def gettitles(self, target):
		titles = []
		fhp = open(target, 'r')
		for raw_inputline in fhp.readlines():
			raw_inputline = raw_inputline.strip()
			titles.extend(raw_inputline.split())
		fhp.close()
		return titles

	def getwordlist(self):
		return self.wordlist

def UserInterface():
	filelist = []
	#print len(sys.argv)
	if len(sys.argv) > 1:
		command = sys.argv[1]
	else:
		print
		print "Hit enter with no input to quit."
		command = raw_input("Query File(no more than 20 chars):")

	while True:
		if command == '':
			return filelist
		else:
			try:
				fhp = open(command, 'r')
			except:
				print "%s does not exists" % command
				print
				print "Hit enter with no input to quit."
				command = raw_input("Query File(no more than 20 chars):")
				continue
	    		
			filelist.append(command)
			fhp.close()
			break
	return filelist
	
#def Processor(doc):
#    getMethod = doc.get
	    	
def RunSearch(searcher, parser, word):
    global logger, time_costing   
    local_parse  = parser.parse
    local_search = searcher.search

    start = datetime.now()
    hits  = local_search(local_parse(word))
    
    #map(Processor, hits)
    for i in xrange(0, hits.length()):
        getMethod = hits.doc(i).get
        getMethod("name"), getMethod("path"), getMethod("contents")

    end = datetime.now()
    during = end - start

    wss = ["[Result]", "[Time]"]
    wss.insert(1, '\t'+ str(hits.length()))
    wss.append('\t'+ str(during)+ '\n')

    logger.writelines(wss)
    time_costing += during.microseconds/1000

if __name__ == '__main__':
	STORE_DIR = "index"
	LOG_FILE = "logging"
	time_costing = 0
	cur_idx = 0
	total_idx = 0

	print 'PyLucene', VERSION, 'Lucene', LUCENE_VERSION
	directory = FSDirectory.getDirectory(STORE_DIR, False)
	searcher = IndexSearcher(directory)
	parser = QueryParser("contents", StandardAnalyzer())
	logger = open(LOG_FILE, 'w')

	ucase = UserCases(UserInterface())
	ticker = Ticker()
	print 'Running Search test...',
	threading.Thread(target=ticker.run).start()
	total_idx = len(ucase.getwordlist())
		
	for word in ucase.getwordlist():
		cur_idx +=1
		RunSearch(searcher, parser, word)

	ticker.tick = False
	searcher.close()
	print
	print "Total (microseconds):%d" % time_costing


 
-------------- next part --------------
I
HAVE
been
thinking
it
over
again
Elizabeth
said
her
uncle
as
they
drove
from
the
town
and
really
upon
serious
consideration
I
am
much
more
inclined
than
I
was
to
judge
as
your
eldest
sister
does
of
the
matter
It
appears
to
me
so
very
unlikely
that
any
young
man
should
form
such
a
design
against
a
girl
who
is
by
no
means
unprotected
or
friendless
and
who
was
actually
staying
in
his
colonel
s
family
that
I
am
strongly
inclined
to
hope
the
best
Could
he
expect
that
her
friends
would
not
step
forward
Could
he
expect
to
be
noticed
again
by
the
regiment
after
such
an
affront
to
Colonel
Forster
His
temptation
is
not
adequate
to
the
risk
Do
you
really
think
so
cried
Elizabeth
brightening
up
for
a
moment
Upon
my
word
said
Mrs
Gardiner
I
begin
to
be
of
your
uncle
s
opinion
It
is
really
too
great
a
violation
of
decency
honour
and
interest
for
him
to
be
guilty
of
it
I
cannot
think
so
very
ill
of
Wickham
Can
you
yourself
Lizzy
so
wholly
give
him
up
as
to
believe
him
capable
of
it
Not
perhaps
of
neglecting
his
own
interest
But
of
every
other
neglect
I
can
believe
him
capable
If
indeed
it
should
be
so
But
I
dare
not
hope
it
Why
should
they
not
go
on
to
Scotland
if
that
had
been
the
case
In
the
first
place
replied
Mr
Gardiner
there
is
no
absolute
proof
that
they
are
not
gone
to
Scotland
Oh
but
their
removing
from
the
chaise
into
an
hackney
coach
is
such
a
presumption
And
besides
no
traces
of
them
were
to
be
found
on
the
Barnet
road
Well
then
supposing
them
to
be
in
London
They
may
be
there
though
for
the
purpose
of
concealment
for
no
more
exceptionable
purpose
It
is
not
likely
that
money
should
be
very
abundant
on
either
side
and
it
might
strike
them
that
they
could
be
more
economically
though
less
expeditiously
married
in
London
than
in
Scotland
But
why
all
this
secrecy
Why
any
fear
of
detection
Why
must
their
marriage
be
private
Oh
no
no
this
is
not
likely
His
most
particular
friend
you
see
by
Jane
s
account
was
persuaded
of
his
never
intending
to
marry
her
Wickham
will
never
marry
a
woman
without
some
money
He
cannot
afford
it
And
what
claims
has
Lydia
what
attractions
has
she
beyond
youth
health
and
good
humour
that
could
make
him
for
her
sake
forgo
every
chance
of
benefiting
himself
by
marrying
well
As
to
what
restraint
the
apprehension
of
disgrace
in
the
corps
might
throw
on
a
dishonourable
elopement
with
her
I
am
not
able
to
judge
for
I
know
nothing
of
the
effects
that
such
a
step
might
produce
But
as
to
your
other
objection
I
am
afraid
it
will
hardly
hold
good
Lydia
has
no
brothers
to
step
forward
and
he
might
imagine
from
my
father
s
behaviour
from
his
indolence
and
the
little
attention
he
has
ever
seemed
to
give
to
what
was
going
forward
in
his
family
that
he
would
do
as
little
and
think
as
little
about
it
as
any
father
could
do
in
such
a
matter
But
can
you
think
that
Lydia
is
so
lost
to
every
thing
but
love
of
him
as
to
consent
to
live
with
him
on
any
other
terms
than
marriage
It
does
seem
and
it
is
most
shocking
indeed
replied
Elizabeth
with
tears
in
her
eyes
that
a
sister
s
sense
of
decency
and
virtue
in
such
a
point
should
admit
of
doubt
But
really
I
know
not
what
to
say
Perhaps
I
am
not
doing
her
justice
But
she
is
very
young
she
has
never
been
taught
to
think
on
serious
subjects
and
for
the
last
half
year
nay
for
a
twelvemonth
she
has
been
given
up
to
nothing
but
amusement
and
vanity
She
has
been
allowed
to
dispose
of
her
time
in
the
most
idle
and
frivolous
manner
and
to
adopt
any
opinions
that
came
in
her
way
Since
the
shire
were
first
quartered
in
Meryton
nothing
but
love
flirtation
and
officers
have
been
in
her
head
She
has
been
doing
every
thing
in
her
power
by
thinking
and
talking
on
the
subject
to
give
greater
what
shall
I
call
it
susceptibility
to
her
feelings
which
are
naturally
lively
enough
And
we
all
know
that
Wickham
has
every
charm
of
person
and
address
that
can
captivate
a
woman
But
you
see
that
Jane
said
her
aunt
does
not
think
so
ill
of
Wickham
as
to
believe
him
capable
of
the
attempt
Of
whom
does
Jane
ever
think
ill
And
who
is
there
whatever
might
be
their
former
conduct
that
she
would
believe
capable
of
such
an
attempt
till
it
were
proved
against
them
But
Jane
knows
as
well
as
I
do
what
Wickham
really
is
We
both
know
that
he
has
been
profligate
in
every
sense
of
the
word
That
he
has
neither
integrity
nor
honour
That
he
is
as
false
and
deceitful
as
he
is
insinuating
And
do
you
really
know
all
this
cried
Mrs
Gardiner
whose
curiosity
as
to
the
mode
of
her
intelligence
was
all
alive
I
do
indeed
replied
Elizabeth
colouring
I
told
you
the
other
day
of
his
infamous
behaviour
to
Mr
Darcy
and
you
yourself
when
last
at
Longbourn
heard
in
what
manner
he
spoke
of
the
man
who
had
behaved
with
such
forbearance
and
liberality
towards
him
And
there
are
other
circumstances
which
I
am
not
at
liberty
which
it
is
not
worth
while
to
relate
but
his
lies
about
the
whole
Pemberley
family
are
endless
From
what
he
said
of
Miss
Darcy
I
was
thoroughly
prepared
to
see
a
proud
reserved
disagreeable
girl
Yet
he
knew
to
the
contrary
himself
He
must
know
that
she
was
amiable
and
unpretending
as
we
have
found
her
But
does
Lydia
know
nothing
of
this
Can
she
be
ignorant
of
what
you
and
Jane
seem
so
well
to
understand
Oh
yes
that
that
is
the
worst
of
all
Till
I
was
in
Kent
and
saw
so
much
both
of
Mr
Darcy
and
his
relation
Colonel
Fitzwilliam
I
was
ignorant
of
the
truth
myself
And
when
I
returned
home
the
shire
was
to
leave
Meryton
in
a
week
or
fortnight
s
time
As
that
was
the
case
neither
Jane
to
whom
I
related
the
whole
nor
I
thought
it
necessary
to
make
our
knowledge
public
for
of
what
use
could
it
apparently
be
to
any
one
that
the
good
opinion
which
all
the
neighbourhood
had
of
him
should
then
be
overthrown
And
even
when
it
was
settled
that
Lydia
should
go
with
Mrs
Forster
the
necessity
of
opening
her
eyes
to
his
character
never
occurred
to
me
That
she
could
be
in
any
danger
from
the
deception
never
entered
my
head
That
such
a
consequence
as
this
should
ensue
you
may
easily
believe
was
far
enough
from
my
thoughts
When
they
all
removed
to
Brighton
therefore
you
had
no
reason
I
suppose
to
believe
them
fond
of
each
other
Not
the
slightest
I
can
remember
no
symptom
of
affection
on
either
side
and
had
any
thing
of
the
kind
been
perceptible
you
must
be
aware
that
ours
is
not
a
family
on
which
it
could
be
thrown
away
When
first
he
entered
the
corps
she
was
ready
enough
to
admire
him
but
so
we
all
were
Every
girl
in
or
near
Meryton
was
out
of
her
senses
about
him
for
the
first
two
months
but
he
never
distinguished
her
by
any
particular
attention
and
consequently
after
a
moderate
period
of
extravagant
and
wild
admiration
her
fancy
for
him
gave
way
and
others
of
the
regiment
who
treated
her
with
more
distinction
again
became
her
favourites
It
may
be
easily
believed
that
however
little
of
novelty
could
be
added
to
their
fears
hopes
and
conjectures
on
this
interesting
subject
by
its
repeated
discussion
no
other
could
detain
them
from
it
long
during
the
whole
of
the
journey
From
Elizabeth
s
thoughts
it
was
never
absent
Fixed
there
by
the
keenest
of
all
anguish
self
reproach
she
could
find
no
interval
of
ease
or
forgetfulness
They
travelled
as
expeditiously
as
possible
and
sleeping
one
night
on
the
road
reached
Longbourn
by
dinner
time
the
next
day
It
was
a
comfort
to
Elizabeth
to
consider
that
Jane
could
not
have
been
wearied
by
long
expectations
The
little
Gardiners
attracted
by
the
sight
of
a
chaise
were
standing
on
the
steps
of
the
house
as
they
entered
the
paddock
and
when
the
carriage
drove
up
to
the
door
the
joyful
surprise
that
lighted
up
their
faces
and
displayed
itself
over
their
whole
bodies
in
a
variety
of
capers
and
frisks
was
the
first
pleasing
earnest
of
their
welcome
Elizabeth
jumped
out
and
after
giving
each
of
them
an
hasty
kiss
hurried
into
the
vestibule
where
Jane
who
came
running
down
stairs
from
her
mother
s
apartment
immediately
met
her
Elizabeth
as
she
affectionately
embraced
her
whilst
tears
filled
the
eyes
of
both
lost
not
a
moment
in
asking
whether
any
thing
had
been
heard
of
the
fugitives
Not
yet
replied
Jane
But
now
that
my
dear
uncle
is
come
I
hope
every
thing
will
be
well
Is
my
father
in
town
Yes
he
went
on
Tuesday
as
I
wrote
you
word
And
have
you
heard
from
him
often
We
have
heard
only
once
He
wrote
me
a
few
lines
on
Wednesday
to
say
that
he
had
arrived
in
safety
and
to
give
me
his
directions
which
I
particularly
begged
him
to
do
He
merely
added
that
he
should
not
write
again
till
he
had
something
of
importance
to
mention
And
my
mother
How
is
she
How
are
you
all
My
mother
is
tolerably
well
I
trust
though
her
spirits
are
greatly
shaken
She
is
up
stairs
and
will
have
great
satisfaction
in
seeing
you
all
She
does
not
yet
leave
her
dressing
room
Mary
and
Kitty
thank
Heaven
are
quite
well
But
you
How
are
you
cried
Elizabeth
You
look
pale
How
much
you
must
have
gone
through
Her
sister
however
assured
her
of
her
being
perfectly
well
and
their
conversation
which
had
been
passing
while
Mr
and
Mrs
Gardiner
were
engaged
with
their
children
was
now
put
an
end
to
by
the
approach
of
the
whole
party
Jane
ran
to
her
uncle
and
aunt
and
welcomed
and
thanked
them
both
with
alternate
smiles
and
tears
When
they
were
all
in
the
drawing
room
the
questions
which
Elizabeth
had
already
asked
were
of
course
repeated
by
the
others
and
they
soon
found
that
Jane
had
no
intelligence
to
give
The
sanguine
hope
of
good
however
which
the
benevolence
of
her
heart
suggested
had
not
yet
deserted
her
she
still
expected
that
it
would
all
end
well
and
that
every
morning
would
bring
some
letter
either
from
Lydia
or
her
father
to
explain
their
proceedings
and
perhaps
announce
the
marriage
Mrs
Bennet
to
whose
apartment
they
all
repaired
after
a
few
minutes
conversation
together
received
them
exactly
as
might
be
expected
with
tears
and
lamentations
of
regret
invectives
against
the
villainous
conduct
of
Wickham
and
complaints
of
her
own
sufferings
and
ill
usage
blaming
every
body
but
the
person
to
whose
ill
judging
indulgence
the
errors
of
her
daughter
must
be
principally
owing
If
I
had
been
able
said
she
to
carry
my
point
of
going
to
Brighton
with
all
my
family
this
would
not
have
happened
but
poor
dear
Lydia
had
nobody
to
take
care
of
her
Why
did
the
Forsters
ever
let
her
go
out
of
their
sight
I
am
sure
there
was
some
great
neglect
or
other
on
their
side
for
she
is
not
the
kind
of
girl
to
do
such
a
thing
if
she
had
been
well
looked
after
I
always
thought
they
were
very
unfit
to
have
the
charge
of
her
but
I
was
over
ruled
as
I
always
am
Poor
dear
child
And
now
here
s
Mr
Bennet
gone
away
and
I
know
he
will
fight
Wickham
wherever
he
meets
him
and
then
he
will
be
killed
and
what
is
to
become
of
us
all
The
Collinses
will
turn
us
out
before
he
is
cold
in
his
grave
and
if
you
are
not
kind
to
us
brother
I
do
not
know
what
we
shall
do
They
all
exclaimed
against
such
terrific
ideas
and
Mr
Gardiner
after
general
assurances
of
his
affection
for
her
and
all
her
family
told
her
that
he
meant
to
be
in
London
the
very
next
day
and
would
assist
Mr
Bennet
in
every
endeavour
for
recovering
Lydia
Do
not
give
way
to
useless
alarm
added
he
though
it
is
right
to
be
prepared
for
the
worst
there
is
no
occasion
to
look
on
it
as
certain
It
is
not
quite
a
week
since
they
left
Brighton
In
a
few
days
more
we
may
gain
some
news
of
them
and
till
we
know
that
they
are
not
married
and
have
no
design
of
marrying
do
not
let
us
give
the
matter
over
as
lost
As
soon
as
I
get
to
town
I
shall
go
to
my
brother
and
make
him
come
home
with
me
to
Gracechurch
Street
and
then
we
may
consult
together
as
to
what
is
to
be
done
Oh
my
dear
brother
replied
Mrs
Bennet
that
is
exactly
what
I
could
most
wish
for
And
now
do
when
you
get
to
town
find
them
out
wherever
they
may
be
and
if
they
are
not
married
already
make
them
marry
And
as
for
wedding
clothes
do
not
let
them
wait
for
that
but
tell
Lydia
she
shall
have
as
much
money
as
she
chuses
to
buy
them
after
they
are
married
And
above
all
things
keep
Mr
Bennet
from
fighting
Tell
him
what
a
dreadful
state
I
am
in
that
I
am
frightened
out
of
my
wits
and
have
such
tremblings
such
flutterings
all
over
me
such
spasms
in
my
side
and
pains
in
my
head
and
such
beatings
at
heart
that
I
can
get
no
rest
by
night
nor
by
day
And
tell
my
dear
Lydia
not
to
give
any
directions
about
her
clothes
till
she
has
seen
me
for
she
does
not
know
which
are
the
best
warehouses
Oh
brother
how
kind
you
are
I
know
you
will
contrive
it
all
But
Mr
Gardiner
though
he
assured
her
again
of
his
earnest
endeavours
in
the
cause
could
not
avoid
recommending
moderation
to
her
as
well
in
her
hopes
as
her
fears
and
after
talking
with
her
in
this
manner
till
dinner
was
on
table
they
left
her
to
vent
all
her
feelings
on
the
housekeeper
who
attended
in
the
absence
of
her
daughters
Though
her
brother
and
sister
were
persuaded
that
there
was
no
real
occasion
for
such
a
seclusion
from
the
family
they
did
not
attempt
to
oppose
it
for
they
knew
that
she
had
not
prudence
enough
to
hold
her
tongue
before
the
servants
while
they
waited
at
table
and
judged
it
better
that
one
only
of
the
household
and
the
one
whom
they
could
most
trust
should
comprehend
all
her
fears
and
solicitude
on
the
subject
In
the
dining
room
they
were
soon
joined
by
Mary
and
Kitty
who
had
been
too
busily
engaged
in
their
separate
apartments
to
make
their
appearance
before
One
came
from
her
books
and
the
other
from
her
toilette
The
faces
of
both
however
were
tolerably
calm
and
no
change
was
visible
in
either
except
that
the
loss
of
her
favourite
sister
or
the
anger
which
she
had
herself
incurred
in
the
business
had
given
something
more
of
fretfulness
than
usual
to
the
accents
of
Kitty
As
for
Mary
she
was
mistress
enough
of
herself
to
whisper
to
Elizabeth
with
a
countenance
of
grave
reflection
soon
after
they
were
seated
at
table
This
is
a
most
unfortunate
affair
and
will
probably
be
much
talked
of
But
we
must
stem
the
tide
of
malice
and
pour
into
the
wounded
bosoms
of
each
other
the
balm
of
sisterly
consolation
Then
perceiving
in
Elizabeth
no
inclination
of
replying
she
added
Unhappy
as
the
event
must
be
for
Lydia
we
may
draw
from
it
this
useful
lesson
that
loss
of
virtue
in
a
female
is
irretrievable
that
one
false
step
involves
her
in
endless
ruin
that
her
reputation
is
no
less
brittle
than
it
is
beautiful
and
that
she
cannot
be
too
much
guarded
in
her
behaviour
towards
the
undeserving
of
the
other
sex
Elizabeth
lifted
up
her
eyes
in
amazement
but
was
too
much
oppressed
to
make
any
reply
Mary
however
continued
to
console
herself
with
such
kind
of
moral
extractions
from
the
evil
before
them
In
the
afternoon
the
two
elder
Miss
Bennets
were
able
to
be
for
half
an
hour
by
themselves
and
Elizabeth
instantly
availed
herself
of
the
opportunity
of
making
many
enquiries
which
Jane
was
equally
eager
to
satisfy
After
joining
in
general
lamentations
over
the
dreadful
sequel
of
this
event
which
Elizabeth
considered
as
all
but
certain
and
Miss
Bennet
could
not
assert
to
be
wholly
impossible
the
former
continued
the
subject
by
saying
But
tell
me
all
and
every
thing
about
it
which
I
have
not
already
heard
Give
me
farther
particulars
What
did
Colonel
Forster
say
Had
they
no
apprehension
of
any
thing
before
the
elopement
took
place
They
must
have
seen
them
together
for
ever
Colonel
Forster
did
own
that
he
had
often
suspected
some
partiality
especially
on
Lydia
s
side
but
nothing
to
give
him
any
alarm
I
am
so
grieved
for
him
His
behaviour
was
attentive
and
kind
to
the
utmost
He
was
coming
to
us
in
order
to
assure
us
of
his
concern
before
he
had
any
idea
of
their
not
being
gone
to
Scotland
when
that
apprehension
first
got
abroad
it
hastened
his
journey
And
was
Denny
convinced
that
Wickham
would
not
marry
Did
he
know
of
their
intending
to
go
off
Had
Colonel
Forster
seen
Denny
himself
Yes
but
when
questioned
by
him
Denny
denied
knowing
any
thing
of
their
plan
and
would
not
give
his
real
opinion
about
it
He
did
not
repeat
his
persuasion
of
their
not
marrying
and
from
that
I
am
inclined
to
hope
he
might
have
been
misunderstood
before
And
till
Colonel
Forster
came
himself
not
one
of
you
entertained
a
doubt
I
suppose
of
their
being
really
married
How
was
it
possible
that
such
an
idea
should
enter
our
brains
I
felt
a
little
uneasy
a
little
fearful
of
my
sister
s
happiness
with
him
in
marriage
because
I
knew
that
his
conduct
had
not
been
always
quite
right
My
father
and
mother
knew
nothing
of
that
they
only
felt
how
imprudent
a
match
it
must
be
Kitty
then
owned
with
a
very
natural
triumph
on
knowing
more
than
the
rest
of
us
that
in
Lydia
s
last
letter
she
had
prepared
her
for
such
a
step
She
had
known
it
seems
of
their
being
in
love
with
each
other
many
weeks
But
not
before
they
went
to
Brighton
No
I
believe
not
And
did
Colonel
Forster
appear
to
think
ill
of
Wickham
himself
Does
he
know
his
real
character
I
must
confess
that
he
did
not
speak
so
well
of
Wickham
as
he
formerly
did
He
believed
him
to
be
imprudent
and
extravagant
And
since
this
sad
affair
has
taken
place
it
is
said
that
he
left
Meryton
greatly
in
debt
but
I
hope
this
may
be
false
Oh
Jane
had
we
been
less
secret
had
we
told
what
we
knew
of
him
this
could
not
have
happened
Perhaps
it
would
have
been
better
replied
her
sister
But
to
expose
the
former
faults
of
any
person
without
knowing
what
their
present
feelings
were
seemed
unjustifiable
We
acted
with
the
best
intentions
Could
Colonel
Forster
repeat
the
particulars
of
Lydia
s
note
to
his
wife
He
brought
it
with
him
for
us
to
see
Jane
then
took
it
from
her
pocket
book
and
gave
it
to
Elizabeth
These
were
the
contents
MY
DEAR
HARRIET
You
will
laugh
when
you
know
where
I
am
gone
and
I
cannot
help
laughing
myself
at
your
surprise
to
morrow
morning
as
soon
as
I
am
missed
I
am
going
to
Gretna
Green
and
if
you
cannot
guess
with
who
I
shall
think
you
a
simpleton
for
there
is
but
one
man
in
the
world
I
love
and
he
is
an
angel
I
should
never
be
happy
without
him
so
think
it
no
harm
to
be
off
You
need
not
send
them
word
at
Longbourn
of
my
going
if
you
do
not
like
it
for
it
will
make
the
surprise
the
greater
when
I
write
to
them
and
sign
my
name
Lydia
Wickham
What
a
good
joke
it
will
be
I
can
hardly
write
for
laughing
Pray
make
my
excuses
to
Pratt
for
not
keeping
my
engagement
and
dancing
with
him
to
night
Tell
him
I
hope
he
will
excuse
me
when
he
knows
all
and
tell
him
I
will
dance
with
him
at
the
next
ball
we
meet
with
great
pleasure
I
shall
send
for
my
clothes
when
I
get
to
Longbourn
but
I
wish
you
would
tell
Sally
to
mend
a
great
slit
in
my
worked
muslin
gown
before
they
are
packed
up
Good
bye
Give
my
love
to
Colonel
Forster
I
hope
you
will
drink
to
our
good
journey
Your
affectionate
friend
LYDIA
BENNET
Oh
thoughtless
thoughtless
Lydia
cried
Elizabeth
when
she
had
finished
it
What
a
letter
is
this
to
be
written
at
such
a
moment
But
at
least
it
shews
that
she
was
serious
in
the
object
of
her
journey
Whatever
he
might
afterwards
persuade
her
to
it
was
not
on
her
side
a
scheme
of
infamy
My
poor
father
how
he
must
have
felt
it
I
never
saw
any
one
so
shocked
He
could
not
speak
a
word
for
full
ten
minutes
My
mother
was
taken
ill
immediately
and
the
whole
house
in
such
confusion
Oh
Jane
cried
Elizabeth
was
there
a
servant
belonging
to
it
who
did
not
know
the
whole
story
before
the
end
of
the
day
I
do
not
know
I
hope
there
was
But
to
be
guarded
at
such
a
time
is
very
difficult
My
mother
was
in
hysterics
and
though
I
endeavoured
to
give
her
every
assistance
in
my
power
I
am
afraid
I
did
not
do
so
much
as
I
might
have
done
But
the
horror
of
what
might
possibly
happen
almost
took
from
me
my
faculties
Your
attendance
upon
her
has
been
too
much
for
you
You
do
not
look
well
Oh
that
I
had
been
with
you
you
have
had
every
care
and
anxiety
upon
yourself
alone
Mary
and
Kitty
have
been
very
kind
and
would
have
shared
in
every
fatigue
I
am
sure
but
I
did
not
think
it
right
for
either
of
them
Kitty
is
slight
and
delicate
and
Mary
studies
so
much
that
her
hours
of
repose
should
not
be
broken
in
on
My
aunt
Phillips
came
to
Longbourn
on
Tuesday
after
my
father
went
away
and
was
so
good
as
to
stay
till
Thursday
with
me
She
was
of
great
use
and
comfort
to
us
all
and
Lady
Lucas
has
been
very
kind
she
walked
here
on
Wednesday
morning
to
condole
with
us
and
offered
her
services
or
any
of
her
daughters
if
they
could
be
of
use
to
us
She
had
better
have
stayed
at
home
cried
Elizabeth
perhaps
she
meant
well
but
under
such
a
misfortune
as
this
one
cannot
see
too
little
of
one
s
neighbours
Assistance
is
impossible
condolence
insufferable
Let
them
triumph
over
us
at
a
distance
and
be
satisfied
She
then
proceeded
to
enquire
into
the
measures
which
her
father
had
intended
to
pursue
while
in
town
for
the
recovery
of
his
daughter
He
meant
I
believe
replied
Jane
to
go
to
Epsom
the
place
where
they
last
changed
horses
see
the
postilions
and
try
if
any
thing
could
be
made
out
from
them
His
principal
object
must
be
to
discover
the
number
of
the
hackney
coach
which
took
them
from
Clapham
It
had
come
with
a
fare
from
London
and
as
he
thought
the
circumstance
of
a
gentleman
and
lady
s
removing
from
one
carriage
into
another
might
be
remarked
he
meant
to
make
enquiries
at
Clapham
If
he
could
any
how
discover
at
what
house
the
coachman
had
before
set
down
his
fare
he
determined
to
make
enquiries
there
and
hoped
it
might
not
be
impossible
to
find
out
the
stand
and
number
of
the
coach
I
do
not
know
of
any
other
designs
that
he
had
formed
but
he
was
in
such
a
hurry
to
be
gone
and
his
spirits
so
greatly
discomposed
that
I
had
difficulty
in
finding
out
even
so
much
as
this
THE
whole
party
were
in
hopes
of
a
letter
from
Mr
Bennet
the
next
morning
but
the
post
came
in
without
bringing
a
single
line
from
him
His
family
knew
him
to
be
on
all
common
occasions
a
most
negligent
and
dilatory
correspondent
but
at
such
a
time
they
had
hoped
for
exertion
They
were
forced
to
conclude
that
he
had
no
pleasing
intelligence
to
send
but
even
of
that
they
would
have
been
glad
to
be
certain
Mr
Gardiner
had
waited
only
for
the
letters
before
he
set
off
When
he
was
gone
they
were
certain
at
least
of
receiving
constant
information
of
what
was
going
on
and
their
uncle
promised
at
parting
to
prevail
on
Mr
Bennet
to
return
to
Longbourn
as
soon
as
he
could
to
the
great
consolation
of
his
sister
who
considered
it
as
the
only
security
for
her
husband
s
not
being
killed
in
a
duel
Mrs
Gardiner
and
the
children
were
to
remain
in
Hertfordshire
a
few
days
longer
as
the
former
thought
her
presence
might
be
serviceable
to
her
nieces
She
shared
in
their
attendance
on
Mrs
Bennet
and
was
a
great
comfort
to
them
in
their
hours
of
freedom
Their
other
aunt
also
visited
them
frequently
and
always
as
she
said
with
the
design
of
cheering
and
heartening
them
up
though
as
she
never
came
without
reporting
some
fresh
instance
of
Wickham
s
extravagance
or
irregularity
she
seldom
went
away
without
leaving
them
more
dispirited
than
she
found
them
All
Meryton
seemed
striving
to
blacken
the
man
who
but
three
months
before
had
been
almost
an
angel
of
light
He
was
declared
to
be
in
debt
to
every
tradesman
in
the
place
and
his
intrigues
all
honoured
with
the
title
of
seduction
had
been
extended
into
every
tradesman
s
family
Every
body
declared
that
he
was
the
wickedest
young
man
in
the
world
and
every
body
began
to
find
out
that
they
had
always
distrusted
the
appearance
of
his
goodness
Elizabeth
though
she
did
not
credit
above
half
of
what
was
said
believed
enough
to
make
her
former
assurance
of
her
sister
s
ruin
still
more
certain
and
even
Jane
who
believed
still
less
of
it
became
almost
hopeless
more
especially
as
the
time
was
now
come
when
if
they
had
gone
to
Scotland
which
she
had
never
before
entirely
despaired
of
they
must
in
all
probability
have
gained
some
news
of
them
Mr
Gardiner
left
Longbourn
on
Sunday
on
Tuesday
his
wife
received
a
letter
from
him
it
told
them
that
on
his
arrival
he
had
immediately
found
out
his
brother
and
persuaded
him
to
come
to
Gracechurch
street
that
Mr
Bennet
had
been
to
Epsom
and
Clapham
before
his
arrival
but
without
gaining
any
satisfactory
information
and
that
he
was
now
determined
to
enquire
at
all
the
principal
hotels
in
town
as
Mr
Bennet
thought
it
possible
they
might
have
gone
to
one
of
them
on
their
first
coming
to
London
before
they
procured
lodgings
Mr
Gardiner
himself
did
not
expect
any
success
from
this
measure
but
as
his
brother
was
eager
in
it
he
meant
to
assist
him
in
pursuing
it
He
added
that
Mr
Bennet
seemed
wholly
disinclined
at
present
to
leave
London
and
promised
to
write
again
very
soon
There
was
also
a
postscript
to
this
effect
I
have
written
to
Colonel
Forster
to
desire
him
to
find
out
if
possible
from
some
of
the
young
man
s
intimates
in
the
regiment
whether
Wickham
has
any
relations
or
connections
who
would
be
likely
to
know
in
what
part
of
the
town
he
has
now
concealed
himself
If
there
were
any
one
that
one
could
apply
to
with
a
probability
of
gaining
such
a
clue
as
that
it
might
be
of
essential
consequence
At
present
we
have
nothing
to
guide
us
Colonel
Forster
will
I
dare
say
do
every
thing
in
his
power
to
satisfy
us
on
this
head
But
on
second
thoughts
perhaps
Lizzy
could
tell
us
what
relations
he
has
now
living
better
than
any
other
person
Elizabeth
was
at
no
loss
to
understand
from
whence
this
deference
for
her
authority
proceeded
but
it
was
not
in
her
power
to
give
any
information
of
so
satisfactory
a
nature
as
the
compliment
deserved
She
had
never
heard
of
his
having
had
any
relations
except
a
father
and
mother
both
of
whom
had
been
dead
many
years
It
was
possible
however
that
some
of
his
companions
in
the
shire
might
be
able
to
give
more
information
and
though
she
was
not
very
sanguine
in
expecting
it
the
application
was
a
something
to
look
forward
to
Every
day
at
Longbourn
was
now
a
day
of
anxiety
but
the
most
anxious
part
of
each
was
when
the
post
was
expected
The
arrival
of
letters
was
the
first
grand
object
of
every
morning
s
impatience
Through
letters
whatever
of
good
or
bad
was
to
be
told
would
be
communicated
and
every
succeeding
day
was
expected
to
bring
some
news
of
importance
But
before
they
heard
again
from
Mr
Gardiner
a
letter
arrived
for
their
father
from
a
different
quarter
from
Mr
Collins
which
as
Jane
had
received
directions
to
open
all
that
came
for
him
in
his
absence
she
accordingly
read
and
Elizabeth
who
knew
what
curiosities
his
letters
always
were
looked
over
her
and
read
it
likewise
It
was
as
follows
MY
DEAR
SIR
I
feel
myself
called
upon
by
our
relationship
and
my
situation
in
life
to
condole
with
you
on
the
grievous
affliction
you
are
now
suffering
under
of
which
we
were
yesterday
informed
by
a
letter
from
Hertfordshire
Be
assured
my
dear
Sir
that
Mrs
Collins
and
myself
sincerely
sympathise
with
you
and
all
your
respectable
family
in
your
present
distress
which
must
be
of
the
bitterest
kind
because
proceeding
from
a
cause
which
no
time
can
remove
No
arguments
shall
be
wanting
on
my
part
that
can
alleviate
so
severe
a
misfortune
or
that
may
comfort
you
under
a
circumstance
that
must
be
of
all
others
most
afflicting
to
a
parent
s
mind
The
death
of
your
daughter
would
have
been
a
blessing
in
comparison
of
this
And
it
is
the
more
to
be
lamented
because
there
is
reason
to
suppose
as
my
dear
Charlotte
informs
me
that
this
licentiousness
of
behaviour
in
your
daughter
has
proceeded
from
a
faulty
degree
of
indulgence
though
at
the
same
time
for
the
consolation
of
yourself
and
Mrs
Bennet
I
am
inclined
to
think
that
her
own
disposition
must
be
naturally
bad
or
she
could
not
be
guilty
of
such
an
enormity
at
so
early
an
age
Howsoever
that
may
be
you
are
grievously
to
be
pitied
in
which
opinion
I
am
not
only
joined
by
Mrs
Collins
but
likewise
by
Lady
Catherine
and
her
daughter
to
whom
I
have
related
the
affair
They
agree
with
me
in
apprehending
that
this
false
step
in
one
daughter
will
be
injurious
to
the
fortunes
of
all
the
others
for
who
as
Lady
Catherine
herself
condescendingly
says
will
connect
themselves
with
such
a
family
And
this
consideration
leads
me
moreover
to
reflect
with
augmented
satisfaction
on
a
certain
event
of
last
November
for
had
it
been
otherwise
I
must
have
been
involved
in
all
your
sorrow
and
disgrace
Let
me
advise
you
then
my
dear
Sir
to
console
yourself
as
much
as
possible
to
throw
off
your
unworthy
child
from
your
affection
for
ever
and
leave
her
to
reap
the
fruits
of
her
own
heinous
offence
I
am
dear
Sir
c
c
Mr
Gardiner
did
not
write
again
till
he
had
received
an
answer
from
Colonel
Forster
and
then
he
had
nothing
of
a
pleasant
nature
to
send
It
was
not
known
that
Wickham
had
a
single
relation
with
whom
he
kept
up
any
connection
and
it
was
certain
that
he
had
no
near
one
living
His
former
acquaintance
had
been
numerous
but
since
he
had
been
in
the
militia
it
did
not
appear
that
he
was
on
terms
of
particular
friendship
with
any
of
them
There
was
no
one
therefore
who
could
be
pointed
out
as
likely
to
give
any
news
of
him
And
in
the
wretched
state
of
his
own
finances
there
was
a
very
powerful
motive
for
secrecy
in
addition
to
his
fear
of
discovery
by
Lydia
s
relations
for
it
had
just
transpired
that
he
had
left
gaming
debts
behind
him
to
a
very
considerable
amount
Colonel
Forster
believed
that
more
than
a
thousand
pounds
would
be
necessary
to
clear
his
expences
at
Brighton
He
owed
a
good
deal
in
the
town
but
his
debts
of
honour
were
still
more
formidable
Mr
Gardiner
did
not
attempt
to
conceal
these
particulars
from
the
Longbourn
family
Jane
heard
them
with
horror
A
gamester
she
cried
This
is
wholly
unexpected
I
had
not
an
idea
of
it
Mr
Gardiner
added
in
his
letter
that
they
might
expect
to
see
their
father
at
home
on
the
following
day
which
was
Saturday
Rendered
spiritless
by
the
ill
success
of
all
their
endeavours
he
had
yielded
to
his
brother
in
law
s
intreaty
that
he
would
return
to
his
family
and
leave
it
to
him
to
do
whatever
occasion
might
suggest
to
be
advisable
for
continuing
their
pursuit
When
Mrs
Bennet
was
told
of
this
she
did
not
express
so
much
satisfaction
as
her
children
expected
considering
what
her
anxiety
for
his
life
had
been
before
What
is
he
coming
home
and
without
poor
Lydia
she
cried
Sure
he
will
not
leave
London
before
he
has
found
them
Who
is
to
fight
Wickham
and
make
him
marry
her
if
he
comes
away
As
Mrs
Gardiner
began
to
wish
to
be
at
home
it
was
settled
that
she
and
her
children
should
go
to
London
at
the
same
time
that
Mr
Bennet
came
from
it
The
coach
therefore
took
them
the
first
stage
of
their
journey
and
brought
its
master
back
to
Longbourn
Mrs
Gardiner
went
away
in
all
the
perplexity
about
Elizabeth
and
her
Derbyshire
friend
that
had
attended
her
from
that
part
of
the
world
His
name
had
never
been
voluntarily
mentioned
before
them
by
her
niece
and
the
kind
of
half
expectation
which
Mrs
Gardiner
had
formed
of
their
being
followed
by
a
letter
from
him
had
ended
in
nothing
Elizabeth
had
received
none
since
her
return
that
could
come
from
Pemberley
The
present
unhappy
state
of
the
family
rendered
any
other
excuse
for
the
lowness
of
her
spirits
unnecessary
nothing
therefore
could
be
fairly
conjectured
from
that
though
Elizabeth
who
was
by
this
time
tolerably
well
acquainted
with
her
own
feelings
was
perfectly
aware
that
had
she
known
nothing
of
Darcy
she
could
have
borne
the
dread
of
Lydia
s
infamy
somewhat
better
It
would
have
spared
her
she
thought
one
sleepless
night
out
of
two
When
Mr
Bennet
arrived
he
had
all
the
appearance
of
his
usual
philosophic
composure
He
said
as
little
as
he
had
ever
been
in
the
habit
of
saying
made
no
mention
of
the
business
that
had
taken
him
away
and
it
was
some
time
before
his
daughters
had
courage
to
speak
of
it
It
was
not
till
the
afternoon
when
he
joined
them
at
tea
that
Elizabeth
ventured
to
introduce
the
subject
and
then
on
her
briefly
expressing
her
sorrow
for
what
he
must
have
endured
he
replied
Say
nothing
of
that
Who
would
suffer
but
myself
It
has
been
my
own
doing
and
I
ought
to
feel
it
You
must
not
be
too
severe
upon
yourself
replied
Elizabeth
You
may
well
warn
me
against
such
an
evil
Human
nature
is
so
prone
to
fall
into
it
No
Lizzy
let
me
once
in
my
life
feel
how
much
I
have
been
to
blame
I
am
not
afraid
of
being
overpowered
by
the
impression
It
will
pass
away
soon
enough
Do
you
suppose
them
to
be
in
London
Yes
where
else
can
they
be
so
well
concealed
And
Lydia
used
to
want
to
go
to
London
added
Kitty
She
is
happy
then
said
her
father
drily
and
her
residence
there
will
probably
be
of
some
duration
Then
after
a
short
silence
he
continued
Lizzy
I
bear
you
no
ill
will
for
being
justified
in
your
advice
to
me
last
May
which
considering
the
event
shews
some
greatness
of
mind
They
were
interrupted
by
Miss
Bennet
who
came
to
fetch
her
mother
s
tea
This
is
a
parade
cried
he
which
does
one
good
it
gives
such
an
elegance
to
misfortune
Another
day
I
will
do
the
same
I
will
sit
in
my
library
in
my
night
cap
and
powdering
gown
and
give
as
much
trouble
as
I
can
or
perhaps
I
may
defer
it
till
Kitty
runs
away
I
am
not
going
to
run
away
Papa
said
Kitty
fretfully
if
I
should
ever
go
to
Brighton
I
would
behave
better
than
Lydia
You
go
to
Brighton
I
would
not
trust
you
so
near
it
as
East
Bourne
for
fifty
pounds
No
Kitty
I
have
at
last
learnt
to
be
cautious
and
you
will
feel
the
effects
of
it
No
officer
is
ever
to
enter
my
house
again
nor
even
to
pass
through
the
village
Balls
will
be
absolutely
prohibited
unless
you
stand
up
with
one
of
your
sisters
And
you
are
never
to
stir
out
of
doors
till
you
can
prove
that
you
have
spent
ten
minutes
of
every
day
in
a
rational
manner
Kitty
who
took
all
these
threats
in
a
serious
light
began
to
cry
Well
well
said
he
do
not
make
yourself
unhappy
If
you
are
a
good
girl
for
the
next
ten
years
I
will
take
you
to
a
review
at
the
end
of
them
-------------- next part --------------
package com.focustech.search.core;

import java.io.File;
import java.util.Collection;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.time.StopWatch;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;

public class Search
{
	private String index_home;
	private String word_file;
	private Analyzer analyzer = new StandardAnalyzer();
	private String[] m_words;
	public static void main(String[] args)
	{
		try
		{
			if (args.length != 2)
			{
				System.out.println("Useage: index-path words-file");
				System.exit(0);
			}
			Search o = new Search(args[0], args[1]);
			o.search();
		} catch (Exception e)
		{
			e.printStackTrace();
		}
	}
	public Search(String indexHome, String wordFile)
	{
		index_home = indexHome;
		word_file = wordFile;
	}
	private void search() throws Exception
	{
		load();
		find();
	}
	private void load() throws Exception
	{
		Collection cols = FileUtils
				.readLines(new File(word_file), "iso-8859-1");
		m_words = (String[]) cols.toArray(new String[0]);
	}
	private void find() throws Exception
	{
		final String[] titles = {"name", "path", "contents"};
		IndexSearcher is = new IndexSearcher(index_home);
		QueryParser qp = new QueryParser("contents", analyzer);
		StopWatch clock = new StopWatch();
		long stamp = 0L;
		long sigil = 0L;

		clock.start();
		for (int i = 0; m_words != null && i < m_words.length; i++)
		{
			//clock.suspend();
			int testonly = 0;
			Query q = qp.parse(m_words[i]);
			Hits h = is.search(q);
			
			clock.suspend();
			System.out.println("\r" + i);
			clock.resume();

			for(int j = 0; j < h.length(); j ++)
			{
				h.doc(j).get("name"); 
				h.doc(j).get("path");
				h.doc(j).get("contens");
				testonly = j; 
			}
			
		}
		clock.stop();
		System.out.println("Total: " + clock.getTime() + "ms.");
	}
}