[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
gnue/integrator samples/sample.gmd src/GIClient...
|
From: |
Jan Ischebeck |
|
Subject: |
gnue/integrator samples/sample.gmd src/GIClient... |
|
Date: |
Sat, 30 Nov 2002 18:25:32 -0500 |
CVSROOT: /cvsroot/gnue
Module name: gnue
Changes by: Jan Ischebeck <address@hidden> 02/11/30 18:25:32
Modified files:
integrator/samples: sample.gmd
integrator/src : GIClient.py GIObjects.py GIParser.py
Log message:
some more preliminary tests (writing and trigger support)
CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/gnue/gnue/integrator/samples/sample.gmd.diff?tr1=1.1&tr2=1.2&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gnue/gnue/integrator/src/GIClient.py.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gnue/gnue/integrator/src/GIObjects.py.diff?tr1=1.4&tr2=1.5&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/gnue/gnue/integrator/src/GIParser.py.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
Patches:
Index: gnue/integrator/samples/sample.gmd
diff -c gnue/integrator/samples/sample.gmd:1.1
gnue/integrator/samples/sample.gmd:1.2
*** gnue/integrator/samples/sample.gmd:1.1 Thu Nov 21 18:48:48 2002
--- gnue/integrator/samples/sample.gmd Sat Nov 30 18:25:32 2002
***************
*** 5,35 ****
<datasource name="in" table="buch" database="gnue"/>
<datasource name="out" table="buchneu" database="gnue"/>
</sources>
! <mapping id="no" title="simple Mapping">
! <process rowmapping="merge|split|simple">
<action type="copy">
! <field type="src" name="author" datasource="in"/>
! <field type="dest" name="author" datasource="out"/>
</action>
! <action type="merge" mergemask="@1 @2">
! <field type="src" name="title" datasource="in"/>
! <field type="src" name="year" datasource="in"/>
! <field type="dest" name="title" datasource="out"/>
</action>
<action type="split" splitmask="(.*):(.*)">
! <field type="src" name="year" datasource="in"/>
! <field type="dest" name="year" datasource="out"/>
! <field type="dest" name="year" datasource="out"/>
</action>
- <trigger type="POST-ROW">
- ...
- </trigger>
<trigger type="PRE-ROW">
! ...
</trigger>
! </process>
</mapping>
! <mapping title="XXX" id="ar">
<submapping>
</submapping>
</mapping>
--- 5,36 ----
<datasource name="in" table="buch" database="gnue"/>
<datasource name="out" table="buchneu" database="gnue"/>
</sources>
! <mapping id="no" title="simple Mapping" source="in" destination="out">
<action type="copy">
! <field type="src" name="author"/>
! <field type="dest" name="author"/>
</action>
! <action type="merge" mergemask="@2 @1">
! <field type="src" name="title"/>
! <field type="src" name="year"/>
! <field type="dest" name="title"/>
</action>
<action type="split" splitmask="(.*):(.*)">
! <field type="src" name="year"/>
! <field type="dest" name="year"/>
! <field type="dest" name="year"/>
</action>
<trigger type="PRE-ROW">
! print "PRE-ROW"
! # print getField('author')
</trigger>
! <trigger type="POST-ROW">
! print "POST-ROW"
! # print getField("author")
! # setField("year","200")
! </trigger>
</mapping>
! <mapping title="XXX" id="ar" source="in" destination="out">
<submapping>
</submapping>
</mapping>
Index: gnue/integrator/src/GIClient.py
diff -c gnue/integrator/src/GIClient.py:1.2 gnue/integrator/src/GIClient.py:1.3
*** gnue/integrator/src/GIClient.py:1.2 Thu Nov 21 18:48:48 2002
--- gnue/integrator/src/GIClient.py Sat Nov 30 18:25:32 2002
***************
*** 49,55 ****
COMMAND = "gnue-integrator"
NAME = "GNUe Integrator"
USAGE = "[options] file"
! COMMAND_OPTIONS = []
SUMMARY = \
"GNUe Integrator is the data integration/migration tool of the GNU
Enterprise system."
--- 49,59 ----
COMMAND = "gnue-integrator"
NAME = "GNUe Integrator"
USAGE = "[options] file"
! COMMAND_OPTIONS = [
! [ 'testing','t','testing',1,None,None,
! _("If you set this flag, Integrator won't write any data to ")+
! _("destination datasources. ")]
! ]
SUMMARY = \
"GNUe Integrator is the data integration/migration tool of the GNU
Enterprise system."
***************
*** 80,87 ****
! # add connections object to top object
! # mapping._connections=self.connections
mapping.run()
--- 84,91 ----
! if self.OPTIONS["testing"]:
! mapping._testonly=1
mapping.run()
Index: gnue/integrator/src/GIObjects.py
diff -c gnue/integrator/src/GIObjects.py:1.4
gnue/integrator/src/GIObjects.py:1.5
*** gnue/integrator/src/GIObjects.py:1.4 Thu Nov 21 18:48:48 2002
--- gnue/integrator/src/GIObjects.py Sat Nov 30 18:25:32 2002
***************
*** 32,63 ****
from gnue.common.GRootObj import GRootObj
from gnue.reports import GREngine,GRExceptions
import GIParser
# Base class for all Integrator objects
class GIObject(GObjects.GObj):
pass
! class GIMappings(GRootObj,GIObject):
def __init__(self, parent=None):
GRootObj.__init__(self, 'mappings', GIParser.getXMLelements,GIParser)
GIObject.__init__(self, parent, type="GIMappings")
self._datasourceDictionary={}
def run(self):
for child in self._children:
if child._type=="GIMapping":
child.run()
!
! class GIMapping(GIObject):
def __init__(self, parent):
GIObject.__init__(self, parent, type="GIMapping")
def run(self):
GDebug.printMesg(5,"Start Mapping %s." % self.title)
! for child in self._children:
! if child._type=="GIProcess":
! child.run()
class GISubMapping(GIObject):
def __init__(self, parent):
--- 32,194 ----
from gnue.common.GRootObj import GRootObj
from gnue.reports import GREngine,GRExceptions
import GIParser
+ from gnue.common.GTrigger import GTriggerExtension
# Base class for all Integrator objects
class GIObject(GObjects.GObj):
pass
! class GIMappings(GRootObj,GIObject,GTriggerExtension):
def __init__(self, parent=None):
+ GTriggerExtension.__init__(self)
GRootObj.__init__(self, 'mappings', GIParser.getXMLelements,GIParser)
GIObject.__init__(self, parent, type="GIMappings")
self._datasourceDictionary={}
+ self._inits = (self.initialize,)
+
+ self.name=""
+ #
+ # New trigger support
+ #
+ self._triggerDictionary = {}
+ self._triggerns={}
+
+ self._validTriggers = { 'ON-STARTUP': 'On-Startup',
+ 'ON-EXIT': 'On-Exit' }
+
+ #self._triggerGlobal = 1
+ self._triggerFunctions = {}
+
+ def initialize(self):
+ self.initTriggerSystem()
+ self._triggerns.update(self._triggerNamespaceTree._globalNamespace)
+
def run(self):
+ self.processTrigger('On-Startup')
+
for child in self._children:
if child._type=="GIMapping":
child.run()
!
! self.processTrigger('On-Exit')
!
!
! class GIMapping(GIObject,GTriggerExtension):
def __init__(self, parent):
+ GTriggerExtension.__init__(self)
GIObject.__init__(self, parent, type="GIMapping")
+ self._inits = [self.initialize]
+ self._validTriggers = {'POST-ROW':'Post-Row',
+ 'PRE-ROW':'Pre-Row',
+ 'POST-PROCESS':'Post-Process',
+ 'PRE-PROCESS':'Pre-Process'}
+
+ #
+ # temporariy hack: remove this functions if trigger namespace is
+ # populated correctly
+ self._triggerGlobal = 1
+ self._triggerFunctions={'setField':{'function':self.triggerSetValue,
+ 'global':1},
+ 'getField':{'function':self.triggerGetValue,
+ 'global':1}
+ }
+
+ def triggerSetValue(self,field,value):
+ self._outputRset.current.setField(field,value)
+
+ def triggerGetValue(self,field):
+ return self._inputRset.current.getField(field)
+
+ # end hack
+
+ def initialize(self):
+ self._sourceFields=[]
+ self._destinationFields=[]
+ self._inputSource = \
+
self._parent._datasourceDictionary[string.lower(self.source)]
+ self._outputSource = \
+
self._parent._datasourceDictionary[string.lower(self.destination)]
+
+
+ def addSourceField(self,name):
+ self._sourceFields.append(name)
+ self._inputSource.referenceField(name, None)
+
+ def addDestinationField(self,name):
+ self._destinationFields.append(name)
+ self._outputSource.referenceField(name, None)
+
def run(self):
GDebug.printMesg(5,"Start Mapping %s." % self.title)
!
! # Prepare Input datasource
! if self._inputSource.hasMaster():
! raise Exception,'InputSources in a Mapping should not have a master'
!
! self._inputRset=self._inputSource.createResultSet({})
!
! # Prepare Output datasource
! if self._outputSource.hasMaster():
! raise Exception,'OutputSources in a Mapping should not have a master'
!
! if not self.append:
! GDebug.printMesg(5,"Append not set: All records in Source %s will be " \
! % self.destination + 'removed !!!')
! # TODO: add code to remove records
! # like self._outputSource.sql('delete from %s;' % tablename)
!
!
! self._outputRset=self._outputSource.createResultSet()
!
! self.processTrigger('Pre-Process')
!
! record=self._inputRset.firstRecord()
! while record:
!
! if not hasattr(self._parent,'_testonly'):
! self._outputRset.insertRecord ()
!
! output={}
!
! self.processTrigger('Pre-Row')
!
! for child in self._children:
! if child._type=="GIAction":
!
! out=child.process(self._inputRset.current)
!
! if out!=None:
! output.update(out)
!
! self.processTrigger('Post-Row')
!
! for field in output.keys():
! # TODO: add some basic output field processing here
!
! if not hasattr(self._parent,'_testonly'):
! self._outputRset.current.setField(field,output[field])
!
!
! if hasattr(self._parent,'_testonly'):
! print output
!
! record=self._inputRset.nextRecord()
!
! self.processTrigger('Post-Process')
!
! self._outputRset.post()
! self._outputSource.commit()
!
! # self._inputRset.close()
! # self._outputRset.close()
!
! # def run(self):
! # for child in self._children:
! # if child._type=="GIProcess":
! # child.run()
class GISubMapping(GIObject):
def __init__(self, parent):
***************
*** 70,97 ****
class GIProcess(GIObject):
def __init__(self, parent):
GIObject.__init__(self, parent, type="GIProcess")
! self.InputSources={}
! self.OutputSources={}
self.InputRset={}
self.OutputRset={}
- def run(self):
- for dts in self.InputSources.keys():
- if not self.InputSources[dts].hasMaster():
- self.InputRset[dts]=self.InputSources[dts].createResultSet({})
- for dts in self.OutputSources.keys():
- if not self.OutputSources[dts].hasMaster():
- self.OutputRset[dts]=self.OutputSources[dts].createEmptyResultSet()
- # print self.InputRset
- for rset in self.InputRset.keys():
- rec=self.InputRset[rset].firstRecord()
- while rec:
- for child in self._children:
- if child._type=="GIAction":
- child.process(self.InputRset[rset].current)
- rec=self.InputRset[rset].nextRecord()
- # print "INPUT:", self.InputSources
- # print "OUTPUT:", self.OutputSources
class GIAction(GIObject):
--- 201,211 ----
class GIProcess(GIObject):
def __init__(self, parent):
GIObject.__init__(self, parent, type="GIProcess")
! self.inputSource=None
! self.outputSource=None
self.InputRset={}
self.OutputRset={}
class GIAction(GIObject):
***************
*** 130,136 ****
if child.type=="src":
out=string.replace(out,'@%s' % inpCount, '%s' %
row.getField(child.name))
inpCount+=1
! print row.getField(child.name),' ',
if hasattr(self,"splitre"):
mymatch=self.splitre.match(out)
--- 244,250 ----
if child.type=="src":
out=string.replace(out,'@%s' % inpCount, '%s' %
row.getField(child.name))
inpCount+=1
! # print row.getField(child.name),' ',
if hasattr(self,"splitre"):
mymatch=self.splitre.match(out)
***************
*** 142,148 ****
out=mymatch.groups()
else:
out = [out]
! print ' -> ',out
class GIField(GIObject):
def __init__(self, parent):
--- 256,272 ----
out=mymatch.groups()
else:
out = [out]
!
! outpFields={}
! outpCount=1
! for child in self._children:
! if child.type=="dest":
! outpFields[child.name]=out[outpCount-1]
! outpCount+=1
!
! return outpFields
!
!
class GIField(GIObject):
def __init__(self, parent):
***************
*** 151,169 ****
def initialize(self):
self._mappings = self.findParentOfType('GIMappings')
! self._process = self.findParentOfType('GIProcess')
! self._datasource = \
! self._mappings._datasourceDictionary[
! string.lower(self.datasource)]
!
! self._datasource.referenceField(self.name, None)
if self.type=="src":
! self._process.InputSources[
! string.lower(self.datasource)]=self._datasource
!
if self.type=="dest":
! self._process.OutputSources[
! string.lower(self.datasource)]=self._datasource
class GITrigger(GIObject):
def __init__(self, parent):
--- 275,287 ----
def initialize(self):
self._mappings = self.findParentOfType('GIMappings')
! self._mapping = self.findParentOfType('GIMapping')
!
if self.type=="src":
! self._mapping.addSourceField(self.name)
!
if self.type=="dest":
! self._mapping.addDestinationField(self.name)
class GITrigger(GIObject):
def __init__(self, parent):
Index: gnue/integrator/src/GIParser.py
diff -c gnue/integrator/src/GIParser.py:1.2 gnue/integrator/src/GIParser.py:1.3
*** gnue/integrator/src/GIParser.py:1.2 Thu Nov 21 18:48:48 2002
--- gnue/integrator/src/GIParser.py Sat Nov 30 18:25:32 2002
***************
*** 72,77 ****
--- 72,78 ----
if xmlElements == None:
import GIObjects
import GISources
+ from gnue.common import GTrigger
xmlElements = {
'mappings': {
***************
*** 81,87 ****
'Attributes': {
'title': {
'Typecast': GTypecast.text,
! 'Default': 'Untitled Menu' } },
'ParentTags': None },
'mapping': {
--- 82,95 ----
'Attributes': {
'title': {
'Typecast': GTypecast.text,
! 'Default': 'Untitled Menu' },
! 'type': {
! 'ValueSet': {
! 'direct': {},
! 'rowmerge': {},
! 'rowsplit': {}},
! 'Default': 'simple',
! 'Typecast': GTypecast.name }},
'ParentTags': None },
'mapping': {
***************
*** 92,98 ****
'Unique': 1,
'Typecast': GTypecast.name },
'title': {
! 'Typecast': GTypecast.text } },
'ParentTags': ('mappings',) },
'sources': {
--- 100,118 ----
'Unique': 1,
'Typecast': GTypecast.name },
'title': {
! 'Typecast': GTypecast.text },
! 'source': {
! 'Required': 1,
! 'Typecast': GTypecast.name },
! 'destination': {
! 'Required': 1,
! 'Typecast': GTypecast.name },
! 'append': {
! 'Required': 0,
! 'Default': 1,
! 'Description':'if set to false, destination datasources will'+\
! ' be cleared before inserting new data.',
! 'Typecast': GTypecast.boolean } },
'ParentTags': ('mappings',) },
'sources': {
***************
*** 100,120 ****
'Attributes': {},
'ParentTags': ('mappings',) },
! 'process': {
! 'BaseClass': GIObjects.GIProcess,
! 'Attributes': {
! 'rowmapping': {
! 'ValueSet': {
! 'merge': {},
! 'split': {},
! 'simple': {}},
! 'Default': 'simple',
! 'Typecast': GTypecast.name }},
! 'ParentTags': ('mapping',) },
'submapping': {
'BaseClass': GIObjects.GISubMapping,
! 'Attributes': {},
'ParentTags': ('mapping',) },
--- 120,147 ----
'Attributes': {},
'ParentTags': ('mappings',) },
! ## 'process': {
! ## 'BaseClass': GIObjects.GIProcess,
! ## 'Attributes': {
! ## 'rowmapping': {
! ## 'ValueSet': {
! ## 'merge': {},
! ## 'split': {},
! ## 'simple': {}},
! ## 'Default': 'simple',
! ## 'Typecast': GTypecast.name }},
! ## 'ParentTags': ('mapping',) },
'submapping': {
'BaseClass': GIObjects.GISubMapping,
! 'Attributes': {
! 'type': {
! # 'Required': 1,
! 'ValueSet': {
! 'child': {},
! 'direct': {},
! 'addition': {}},
! 'default':'direct'}},
'ParentTags': ('mapping',) },
***************
*** 140,146 ****
# 'Required': 1,
'Default':'@1',
'Typecast': GTypecast.name }},
! 'ParentTags': ('process',) },
'field': {
'BaseClass': GIObjects.GIField,
--- 167,173 ----
# 'Required': 1,
'Default':'@1',
'Typecast': GTypecast.name }},
! 'ParentTags': ('process','mapping') },
'field': {
'BaseClass': GIObjects.GIField,
***************
*** 156,174 ****
'datasource': {
# 'Required': 1,
'Typecast': GTypecast.name }},
! 'ParentTags': ('merge',) },
-
- 'trigger': {
- 'BaseClass': GIObjects.GITrigger,
- 'Attributes': {
- 'type': {
- 'Required': 1,
- 'Typecast': GTypecast.name },
- 'todts': {
- # 'Required': 1,
- 'Typecast': GTypecast.name }},
- 'ParentTags': ('rowmapping','process','mapping') }
};
#
--- 183,190 ----
'datasource': {
# 'Required': 1,
'Typecast': GTypecast.name }},
! 'ParentTags': ('merge',) }
};
#
***************
*** 182,187 ****
--- 198,212 ----
#'cparam': {
# 'BaseClass': GFObjects.GFCParam }
}))
+
+ #
+ # Add trigger elements
+ #
+ xmlElements.update(
+ GTrigger.getXMLelements(
+ updates={'trigger':{
+ 'ParentTags': ('mapping','submapping','action') },
+ }))
return xmlElements
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- gnue/integrator samples/sample.gmd src/GIClient...,
Jan Ischebeck <=