diff --git a/MANIFEST.in b/MANIFEST.in
index f744ce2..deabbd7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,6 @@
include *.txt
recursive-include collective/blueprint/jsonmigrator *
recursive-include export_scripts *
+global-include *.zcml
global-exclude *pyc
diff --git a/collective/blueprint/jsonmigrator/blueprint.py b/collective/blueprint/jsonmigrator/blueprint.py
index 19fe7fc..91be7cf 100644
--- a/collective/blueprint/jsonmigrator/blueprint.py
+++ b/collective/blueprint/jsonmigrator/blueprint.py
@@ -1,10 +1,13 @@
import time
+import tempfile
import os
+import os.path
import simplejson
import logging
import transaction
-
+import shutil
+from PIL import Image
from DateTime import DateTime
from Acquisition import aq_base
from ZODB.POSException import ConflictError
@@ -12,6 +15,9 @@
from zope.interface import implements
from zope.interface import classProvides
+from plone.i18n.normalizer import idnormalizer
+
+
from collective.transmogrifier.interfaces import ISectionBlueprint
from collective.transmogrifier.interfaces import ISection
from collective.transmogrifier.utils import Matcher
@@ -21,11 +27,22 @@
from Products.CMFCore.utils import getToolByName
from Products.Archetypes.interfaces import IBaseObject
+from Products.Archetypes.BaseUnit import BaseUnit
from AccessControl.interfaces import IRoleManager
+from Products.PloneArticle.interfaces import IPloneArticle
+
+## linguaplone migration
+HAVE_LP = None
+try:
+ from Products.LinguaPlone.interfaces import ITranslatable
+ HAVE_LP = True
+except ImportError:
+ HAVE_LP = False
DATAFIELD = '_datafield_'
STATISTICSFIELD = '_statistics_field_prefix_'
+logger = logging.getLogger('collective.blueprint.jsonmigrator')
class JSONSource(object):
""" """
@@ -45,6 +62,7 @@ def __init__(self, transmogrifier, name, options, previous):
raise Exception, 'Path ('+str(self.path)+') does not exists.'
self.datafield_prefix = options.get('datafield-prefix', DATAFIELD)
+ self.datafield_separator = options.get('datafield-separator', None)
def __iter__(self):
for item in self.previous:
@@ -57,14 +75,27 @@ def __iter__(self):
for item2 in sorted([int(j[:-5])
for j in os.listdir(os.path.join(self.path, str(item3)))
if j.endswith('.json')]):
-
- f = open(os.path.join(self.path, str(item3), str(item2)+'.json'))
- item = simplejson.loads(f.read())
+ json_file_path = os.path.join(self.path, str(item3),
+ str(item2)+'.json')
+ f = open(json_file_path)
+ try:
+ item = simplejson.loads(f.read())
+ except:
+ logger.exception('error in reading %s' % json_file_path)
+ item['_json_file_path'] = json_file_path
f.close()
-
for key in item.keys():
if key.startswith(self.datafield_prefix):
- item[key] = os.path.join(self.path, item[key])
+
+ if self.datafield_separator:
+
+ item[key]['path'] = item[key]['path'].replace(\
+ self.datafield_separator,
+ os.path.sep)
+ #file_name = os.path.join(os.path.dirname(item[key]['']
+ # os.path.basename(item[key]['path']
+ item[key]['path'] = os.path.join(self.path,
+ item[key]['path'])
yield item
@@ -156,10 +187,6 @@ def __iter__(self):
stat += 'TOTAL TIME: %d; ' % (now - self.stats['START_TIME'])
stat += 'STEP TIME: %d; ' % (now - self.stats['TIME_LAST_STEP'])
self.stats['TIME_LAST_STEP'] = now
- stat += 'EXISTED: %d; ADDED: %d; NOT-ADDED: %d' % (
- self.stats['EXISTED'],
- self.stats['ADDED'],
- self.stats['NOT-ADDED'])
logging.warning(stat)
@@ -302,7 +329,6 @@ def __iter__(self):
obj = self.context.unrestrictedTraverse(item[pathkey].lstrip('/'), None)
if obj is None: # path doesn't exist
yield item; continue
-
if IBaseObject.providedBy(obj):
if getattr(aq_base(obj), '_delProperty', False):
for prop in item[propertieskey]:
@@ -335,6 +361,7 @@ def __init__(self, transmogrifier, name, options, previous):
self.options = options
self.previous = previous
self.context = transmogrifier.context
+ self.acl_users = getToolByName(self.context, 'acl_users')
self.memtool = getToolByName(self.context, 'portal_membership')
if 'path-key' in options:
@@ -353,7 +380,6 @@ def __iter__(self):
for item in self.previous:
pathkey = self.pathkey(*item.keys())[0]
ownerkey = self.ownerkey(*item.keys())[0]
-
if not pathkey or not ownerkey or \
ownerkey not in item: # not enough info
yield item; continue
@@ -364,23 +390,16 @@ def __iter__(self):
if IBaseObject.providedBy(obj):
- if item[ownerkey][0] and item[ownerkey][1]:
+ if item[ownerkey]:
try:
- obj.changeOwnership(self.memtool.getMemberById(item[ownerkey][1]))
+ obj.changeOwnership(self.acl_users.getUserById(item[ownerkey]))
except Exception, e:
raise Exception('ERROR: %s SETTING OWNERSHIP TO %s' % (str(e), item[pathkey]))
try:
- obj.manage_setLocalRoles(item[ownerkey][1], ['Owner'])
+ obj.manage_setLocalRoles(item[ownerkey], ['Owner'])
except Exception, e:
raise Exception('ERROR: %s SETTING OWNERSHIP2 TO %s' % (str(e), item[pathkey]))
-
- elif not item[ownerkey][0] and item[ownerkey][1]:
- try:
- obj._owner = item[ownerkey][1]
- except Exception, e:
- raise Exception('ERROR: %s SETTING __OWNERSHIP TO %s' % (str(e), item[pathkey]))
-
yield item
@@ -472,14 +491,247 @@ def __iter__(self):
yield item; continue
if IRoleManager.providedBy(obj):
+
+ if self.options.get('erasebefore'):
+ obj.__ac_local_roles__ = {}
for principal, roles in item[roleskey].items():
if roles:
+ if principal.startswith(u'group_'):
+ principal = idnormalizer.normalize(principal)
obj.manage_addLocalRoles(principal, roles)
- obj.reindexObjectSecurity()
+ obj.reindexObjectSecurity()
+
+ yield item
+
+
+class LinguaRelation(object):
+ """ an section about linguaplone """
+
+ classProvides(ISectionBlueprint)
+ implements(ISection)
+
+ def __init__(self, transmogrifier, name, options, previous):
+ self.transmogrifier = transmogrifier
+ self.name = name
+ self.options = options
+ self.previous = previous
+ self.context = transmogrifier.context
+
+ if 'path-key' in options:
+ pathkeys = options['path-key'].splitlines()
+ else:
+ pathkeys = defaultKeys(options['blueprint'], name, 'path')
+ self.pathkey = Matcher(*pathkeys)
+
+
+
+ def __iter__(self):
+ for item in self.previous:
+
+ pathkey = self.pathkey(*item.keys())[0]
+
+ if not pathkey: # not enough info
+ yield item; continue
+ if not HAVE_LP: ## not LinguaPlone
+ yield item; continue
+ #if 'mission' in item[pathkey]:
+ # import pdb;pdb.set_trace();
+
+ obj = self.context.unrestrictedTraverse(item[pathkey].lstrip('/'), None)
+ if obj is None: # path doesn't exist
+ yield item; continue
+ if obj.getLanguage() != item['language']:
+ obj.setLanguage(item['language'])
+
+
+ if not ITranslatable.providedBy(obj):
+ yield item; continue ## not a linguaplone object
+ else:
+ canonical_path = item.get('_canonical')
+ language = item.get('language','')
+ if not canonical_path:
+ yield item; continue
+ try:
+ canonical = self.context.unrestrictedTraverse(canonical_path.lstrip('/'), None)
+ except:
+ yield item; continue
+ try:
+ if not canonical.hasTranslation(language):
+ canonical.addTranslationReference(obj)
+ yield item; continue
+ except:
+ yield item; continue
+
+class PloneArticleFields(object):
+ """ updata data for plonearticle fields """
+
+ classProvides(ISectionBlueprint)
+ implements(ISection)
+ def __init__(self, transmogrifier, name, options, previous):
+ self.transmogrifier = transmogrifier
+ self.name = name
+ self.options = options
+ self.previous = previous
+ self.context = transmogrifier.context
+
+ if 'path-key' in options:
+ pathkeys = options['path-key'].splitlines()
+ else:
+ pathkeys = defaultKeys(options['blueprint'], name, 'path')
+ self.pathkey = Matcher(*pathkeys)
+ self.datafield_separator = options.get('datafield-separator', None)
+
+ def __iter__(self):
+ for item in self.previous:
+ pathkey = self.pathkey(*item.keys())[0]
+ if not pathkey: # not enough info
+ yield item; continue
+ obj = self.context.unrestrictedTraverse(item[pathkey].lstrip('/'), None)
+ if obj is None: # path doesn't exist
+ yield item; continue
+
+ def getUnit(x, field_name):
+ name = x['id'][0]
+ f_path = x[field_name][0]['data']
+ x = x[field_name][0]
+ if self.datafield_separator:
+ f_path = f_path.replace(self.datafield_separator,
+ os.path.sep)
+ f_name = os.path.basename(f_path)
+ f_path = os.path.join(os.path.dirname(\
+ item['_json_file_path']),
+ f_name)
+
+ ###
+ ## type2png = image/x-ms-bmp
+ value = ''
+ if x.get('content_type','') in self.options.get('type2png',''):
+ path = tempfile.mkdtemp()
+ img = Image.open(f_path)
+ new_path = os.path.join(path,'image.png')
+ img.save(new_path)
+ f = open(new_path, mode = 'rb')
+ value = f.read()
+ x['content_type'] = 'image/png'
+ ext = os.path.splitext(x.get('filename', ''))[-1]
+ x['filename'] = x.get('filename','').replace(ext, '.png')
+ try:
+ os.unlink(path)
+ except:
+ pass
+ else:
+ f = open(f_path, mode = 'rb')
+ value = f.read()
+
+
+
+ unit = BaseUnit(name = name,
+ file = value,
+ mimetype = x.get('content_type', ''),
+ filename = x.get('filename', ''),
+ instance = obj)
+ f.close()
+ return unit
+
+ def getReferencedContent(x):
+ path = x['referencedContent'][0]
+ ## we try to get content
+ try:
+ refobj = self.context.restrictedTraverse(path)
+ return (refobj.UID(),{})
+ except:
+ item['_error'] = item['_json_file_path']
+ logger.exception('we cant set referencedContent for %s' % path)
+
+
+ if IPloneArticle.providedBy(obj):
+
+ if '_plonearticle_images' in item and \
+ len(item['_plonearticle_images']):
+ for (i, x) in enumerate(item['_plonearticle_images']) :
+ if 'attachedImage' in x:
+
+ unit = getUnit(x,'attachedImage')
+ item['_plonearticle_images'][i]['attachedImage']=\
+ (unit,{})
+ elif 'referencedContent' in x:
+
+ item['_plonearticle_images'][i]['referencedContent']=getReferencedContent(x)
+ try:
+ obj.getField('images').set(obj,item['_plonearticle_images'])
+ except:
+ item['_error'] = item['_json_file_path']
+ #import pdb;pdb.set_trace();
+ logger.exception('cannot set images for %s %s' % \
+ (item['_path'],
+ item['_json_file_path'])
+ )
+ if '_plonearticle_attachments' in item and\
+ len(item['_plonearticle_attachments']):
+
+ for (i, x) in enumerate(item['_plonearticle_attachments']):
+ if 'attachedFile' in x:
+ unit = getUnit(x,'attachedFile')
+ item['_plonearticle_attachments'][i]['attachedFile'] =\
+ (unit,{})
+ elif 'referencedContent' in x:
+ item['_plonearticle_attachments'][i]['referencedContent']=getReferencedContent(x)
+ try:
+ obj.getField('files').set(obj,
+ item['_plonearticle_attachments'])
+ except:
+ item['_error'] = item['_json_file_path']
+ #import pdb;pdb.set_trace();
+ logger.exception('cannot set files for %s %s' % \
+ (item['_path'],
+ item['_json_file_path'])
+ )
+ if '_plonearticle_refs' in item and \
+ len(item['_plonearticle_refs']):
+ for (i, x) in enumerate(item['_plonearticle_refs']):
+ if 'referencedContent' in x:
+ item['_plonearticle_refs'][i]['referencedContent']=getReferencedContent(x)
+ try:
+ obj.getField('links').set(obj,
+ item['_plonearticle_refs'])
+ except:
+
+ item['_error'] = item['_json_file_path']
+ logger.exception('cannot set links for %s %s' % \
+ (item['_path'],
+ item['_json_file_path'])
+ )
yield item
+class ReportError(object):
+ """ """
+ classProvides(ISectionBlueprint)
+ implements(ISection)
+ def __init__(self, transmogrifier, name, options, previous):
+ self.transmogrifier = transmogrifier
+ self.name = name
+ self.options = options
+ self.previous = previous
+ self.context = transmogrifier.context
+ path = resolvePackageReferenceOrFile(options['path'])
+ self.json = resolvePackageReferenceOrFile(options['json'])
+ self.error_file = open(path,'w')
+
+ def __iter__(self):
+ for item in self.previous:
+ if '_error' in item:
+ self.error_file.write(item['_error'] + "\n")
+ #shutil.copy(item['_error'], self.json)
+ path = os.path.dirname(item['_error'])
+ for x in (x for x in os.listdir(path) \
+ if x.startswith(os.path.basename(item['_error']))):
+ shutil.copy(os.path.join(path, x), self.json)
+
+ yield item
+
+
class DataFields(object):
""" """
@@ -498,33 +750,47 @@ def __init__(self, transmogrifier, name, options, previous):
else:
pathkeys = defaultKeys(options['blueprint'], name, 'path')
self.pathkey = Matcher(*pathkeys)
+
self.datafield_prefix = options.get('datafield-prefix', DATAFIELD)
def __iter__(self):
for item in self.previous:
+
pathkey = self.pathkey(*item.keys())[0]
-
if not pathkey: # not enough info
yield item; continue
-
obj = self.context.unrestrictedTraverse(item[pathkey].lstrip('/'), None)
if obj is None: # path doesn't exist
yield item; continue
-
+
if IBaseObject.providedBy(obj):
for key in item.keys():
if not key.startswith(self.datafield_prefix):
continue
- if not os.path.exists(item[key]):
+ if not os.path.exists(item[key].get('path','')):
continue
fieldname = key[len(self.datafield_prefix):]
field = obj.getField(fieldname)
- f = open(item[key])
+ f = open(item[key]['path'],mode='rb')
value = f.read()
+ unit = BaseUnit(name = fieldname,
+ file = value,
+ mimetype = item[key].get('content_type',''),
+ filename = item[key].get('filename',''),
+ instance = obj
+ )
f.close()
if len(value) != len(field.get(obj)):
- field.set(obj, value)
-
+ try:
+ field.set(obj, unit)
+ except:
+ item['_error'] = item['_json_file_path']
+ logger.exception('cannot set file(%s) for %s %s' % \
+ (fieldname,
+ item['_path'],
+ item['_json_file_path'])
+ )
+
yield item
diff --git a/collective/blueprint/jsonmigrator/configure.zcml b/collective/blueprint/jsonmigrator/configure.zcml
index 519d3d5..d074ae7 100644
--- a/collective/blueprint/jsonmigrator/configure.zcml
+++ b/collective/blueprint/jsonmigrator/configure.zcml
@@ -57,4 +57,21 @@
name="collective.blueprint.jsonmigrator.datafields"
/>
+
+
+
+
+
+
+
+
diff --git a/docs/datafields.rst b/docs/datafields.rst
index aa2bb08..0696fbd 100644
--- a/docs/datafields.rst
+++ b/docs/datafields.rst
@@ -16,6 +16,12 @@ Expected data structure in pipeline:
* **_path**: path to object on which we want to change local roles.
* **_datafield_**: field which needs to store data
+Option configuration:
+
+ * datafield-prefix : for changing the prefix (by default _datafield_)
+ * path-key : for changing the path key
+ * datafield-separator : for changing separator of prefix
+
Example
-------
@@ -38,5 +44,10 @@ Data in pipeline::
{
"_path": "/Plone/index_html",
- "_datafield_attachment": "0/1.json-file-1",
+ "_datafield_attachment": {"filename": "DAF.jpg",
+ "content_type": "image/jpeg",
+ "path": "0\\20.json-file-1",
+ "height": 605,
+ "size": 63912,
+ }
}
diff --git a/docs/index.rst b/docs/index.rst
index 60b0499..9ce440b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,7 +9,7 @@ List of blueprints built around ``collective.blueprints.jsonmigrator.source``
with purpose of providing flexible infrastructure to do migrations in Plone.
In source of this package in ``exports_scripts`` directory is also a helping
-export script ``plone2.0_export.py`` which provides a external method
+export script :doc:`plone2.0_export` which provides a external method
``export_plone20`` to export data from Plone 2.0 (script might also work with
higher versions of plone 2.1, 2.5, but was not tested) in format that is
suitable for ``collective.blueprints.jsonmigrator.source`` blueprint.
@@ -31,6 +31,9 @@ And if you might forgot, migration is a bitch ... so have fun :P
owner
ac_local_roles
datafields
+ plonearticle
+ linguarelation
.. _`collective.transmogrifier`: http://pypi.python.org/pypi/collective.transmogrifier
.. _`Plone`: http://plone.org
+
diff --git a/docs/jsonsource.rst b/docs/jsonsource.rst
index e336be7..d578956 100644
--- a/docs/jsonsource.rst
+++ b/docs/jsonsource.rst
@@ -11,6 +11,12 @@ Parameters
Also possible to specify in ``some.package:path/to/json/directory`` way.
+:path-separator:
+ os path separator use in json file (in case of json file is created on windows)
+
+:datafield-prefix:
+ prefix for indicate file fields prefix. Path is transformed by this blue print
+
Example
-------
@@ -23,6 +29,8 @@ Configuration::
[source]
blueprint = collective.blueprint.jsonmigrator.source
path = some.package:/path/to/json/dir
+ path-separator = \
+ datafield-prefix = _data_
JSON files structure::
@@ -43,4 +51,7 @@ JSON file::
"_path": "/Plone/front-page",
"_type": "Document",
...
+ "_data_backgroundImage": {
+ "path": "0\\20.json-file-1",
+ },
}
diff --git a/docs/linguarelation.rst b/docs/linguarelation.rst
new file mode 100644
index 0000000..1f5ebc0
--- /dev/null
+++ b/docs/linguarelation.rst
@@ -0,0 +1,38 @@
+``collective.blueprint.jsonmigrator.linguarelation``
+====================================================
+
+Set linguaplone relaation between contents.
+
+Configuration options
+---------------------
+
+
+Expected data structure in pipeline:
+
+ * **_canonical**: path of canonical object
+
+
+Example
+-------
+
+This example will try to store content of ``0/1.json-file-1``
+
+Configuration::
+
+ [tranmogrifier]
+ pipeline =
+ source
+ plonearticle
+
+ ...
+
+ [datafields]
+ blueprint = collective.blueprint.jsonmigrator.linguarelation
+
+Data in pipeline::
+
+ {
+ "_path": "/Plone/index_html-fr",
+ "_canonical": "/Plone/index_html",
+
+ }
diff --git a/docs/owner.rst b/docs/owner.rst
index 309e44f..381e8a3 100644
--- a/docs/owner.rst
+++ b/docs/owner.rst
@@ -32,10 +32,8 @@ Data in pipeline::
{
"_path": "/Plone/index_html",
- "_owner": [
- 1,
- "admin"
- ],
+ "_owner": "admin"
+
}
diff --git a/docs/permission_mapping.rst b/docs/permission_mapping.rst
index 901f48e..77abf6a 100644
--- a/docs/permission_mapping.rst
+++ b/docs/permission_mapping.rst
@@ -25,7 +25,7 @@ Configuration::
...
- [mimetype]
+ [permission_mapping]
blueprint = collective.blueprint.jsonmigrator.permission_mapping
Data in pipeline::
diff --git a/docs/plone2.0_export.rst b/docs/plone2.0_export.rst
new file mode 100644
index 0000000..e1a2ce9
--- /dev/null
+++ b/docs/plone2.0_export.rst
@@ -0,0 +1,181 @@
+``plone2.0_export.py``
+======================
+
+Export data from an old plone site.
+
+Installation
+------------
+
+* Create an external method in your plone site.
+
+ - Copy collective.blueprint.jsonmigrator/export_scripts/plone2.0_export.pt in *INSTANCE/Extensions* directory
+ - Connect to ZMI
+ - Add an External Method, and fill out the form with ::
+
+ id = your_id
+ module name = plone_2.0export
+ method = export_plone20
+
+* Create an jsonmigrator.ini in order to configure export process.
+
+
+Syntax of configuration
+-----------------------
+
+
+Options
++++++++
+
+ * In DEFAULT section
+
+ - HOMEDIR => where we create json file. This directory must exists !! Each time that export process is invoked, an new folder is created . In each folder created , every 1000 objects created, script create an new folder. The directory struture look like that::
+
+ HOMEDIR
+ |_ _
+ |_ 0
+ |_ 1.json
+ |_ 2.json
+ |_ ...
+ |_ 999.json
+ |_ 1
+ |_ 1000.json
+ |_ 1001.json
+ |_ ...
+ |_ 1999.json
+ ....
+
+ You can have also file name loke xxx.json-file-x . This is binary file of exported content.
+
+ - CLASSNAME_TO_SKIP_LAUD => This is a list of classname. Object of this classname where are skip by the export process
+
+ - CLASSNAME_TO_SKIP => This is a list of classname. Object of this classname where are skip by the export process
+
+ - ID_TO_SKIP => This is a list of id object . Object wich id is equal to an member of this list is skipping of the process.
+
+ - NON_FOLDERISH_CLASSNAME => This is a list of classname. Object of this classname are considered as non folderish content.
+
+ - JUST_TREAT_WAPPER => If true CLASSNAME_TO_SKIP_LAUD and CLASSNAME_TO_SKIP have no effect. Just object that are mapping in CLASSNAME_TO_WAPPER_MAP are treated
+
+ - MAX_CACHE_DB => a int number that indicate when the process purge the zodb cache (avoid memory error)
+
+ * In CLASSNAME_TO_WAPPER_MAP
+
+ - ClassName=Wrapper => you configure the export wrapper use for object of ClassName
+
+
+Example
++++++++
+
+::
+
+ [DEFAULT]
+ HOMEDIR=c:\dt\plone2.1\export
+ JUST_TREAT_WAPPER=True
+ NON_FOLDERISH_CLASSNAME=DPLDTArticle
+ DPLDTIssue
+ DPLDTPerformance
+ DPLDTTraining
+ MAX_CACHE_DB=250
+
+ [CLASSNAME_TO_WAPPER_MAP]
+ LargePloneFolder=BaseWrapper
+ Folder=BaseWrapper
+ PloneSite=BaseWrapper
+ PloneFolder=BaseWrapper
+ Document=DocumentWrapper
+ File=FileWrapper
+ YourSpecificContentType=ArchetypesWrapper
+
+
+Existing Wrapper
+++++++++++++++++
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: BaseWrapper
+ :end-before: def
+
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: DocumentWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: I18NFolderWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: LinkWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: NewsItemWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ListCriteriaWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: StringCriteriaWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: SortCriteriaWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: DateCriteriaWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: FileWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ImageWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: EventWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ArchetypesWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: I18NLayerWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: Article322Wrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ArticleWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ZPhotoWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ZPhotoSlidesWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ContentPanels
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: LocalFSWrapper
+ :end-before: def
+
+ .. literalinclude:: ../export_scripts/plone2.0_export.py
+ :pyobject: ZopeObjectWrapper
+ :end-before: def
+
+
+
+
+
diff --git a/docs/plonearticle.rst b/docs/plonearticle.rst
new file mode 100644
index 0000000..22b9553
--- /dev/null
+++ b/docs/plonearticle.rst
@@ -0,0 +1,114 @@
+``collective.blueprint.jsonmigrator.plonearticle``
+==================================================
+
+Update images, files, links for plone article contents.
+
+Configuration options
+---------------------
+
+datafield-separator : os separator in case that export is provided by windows system
+
+Expected data structure in pipeline:
+
+ * **_plonearticle_attachments**: information of attached files
+ * **_plonearticle_refs**: information of attached refs
+ * **_plonearticle_images** : information of attached images
+
+Option configuration:
+
+ * datafield-separator : src os separator
+ * path-key : for changing the path key
+
+Example
+-------
+
+This example will try to store content of ``0/1.json-file-1``
+
+Configuration::
+
+ [tranmogrifier]
+ pipeline =
+ source
+ plonearticle
+
+ ...
+
+ [datafields]
+ blueprint = collective.blueprint.jsonmigrator.plonearticle
+
+Data in pipeline::
+
+ {
+ "_path": "/Plone/index_html",
+ "_plonearticle_refs": [
+ {
+ "description": [
+ "Missions",
+ {}
+ ],
+ "referencedContent": [
+ "125d3b5fd50e0da288bfb1d0751a60f7",
+ {}
+ ],
+ "id": [
+ "linkproxy.2011-04-10.5244530114",
+ {}
+ ],
+ "title": [
+ "missions",
+ {}
+ ]
+ }
+ ],
+ "_plonearticle_attachments": [
+ {
+ "attachedFile": [
+ {
+
+ "filename": "Voeux_JPA_VF.doc",
+ "content_type": "application/msword",
+ "data": "0\\1.json-file-1",
+ "size": 29184
+ },
+ {}
+ ],
+ "description": [
+ "",
+ {}
+ ],
+ "id": [
+ "fileproxy.2011-04-10.5244535753",
+ {}
+ ],
+ "title": [
+ "VOEUX 2009 DE J.P AGON",
+ {}
+ ]
+ },
+ {
+ "attachedFile": [
+ {
+ "filename": "IMG_0026 1.JPG",
+ "content_type": "image/jpeg",
+ "data": "0\\1.json-file-2",
+ "size": 1228698
+ },
+ {}
+ ],
+ "description": [
+ "",
+ {}
+ ],
+ "id": [
+ "fileproxy.2011-04-10.5244539481",
+ {}
+ ],
+ "title": [
+ "File.doc",
+ {}
+ ]
+ }
+ ],
+
+
+ }
diff --git a/export_scripts/plone2.0_export.py b/export_scripts/plone2.0_export.py
index a5d3887..966c34b 100644
--- a/export_scripts/plone2.0_export.py
+++ b/export_scripts/plone2.0_export.py
@@ -9,31 +9,143 @@
###############################################################################
import os
+import re
import shutil
+import ConfigParser
+### DEPENDENCY 2.0.0 for python2.3
import simplejson
+
from datetime import datetime
from Acquisition import aq_base
from Products.CMFCore.utils import getToolByName
+from App.config import getConfiguration
+CONFIG = ConfigParser.SafeConfigParser()
+CONFIG.optionxform = str
+import logging
+logger = logging.getLogger('plone20_export')
+
+PAV3_MODEL_RE = re.compile(r'plonearticle_model([\d]*)')
+
+try:
+ #import pdb;pdb.set_trace();
+ CONFIG.readfp(open(os.path.join(getConfiguration().instancehome,
+ 'jsonmigrator.ini')))
+except:
+ logger.exception('Please specify ini file jsonmigrator.ini')
+ logger.warning('Please specify ini file jsonmigrator.ini in your %s' \
+ % getConfiguration().instancehome)
COUNTER = 1
-HOMEDIR = '/Users/rok/Projects/yaco/unex_exported_data'
-CLASSNAME_TO_SKIP_LAUD = ['ControllerPythonScript',
- 'ControllerPageTemplate', 'ControllerValidator', 'PythonScript', 'SQL', 'Connection',
- 'ZetadbScript', 'ExternalMethod', 'ZetadbSqlInsert', 'ZetadbMysqlda', 'SiteRoot',
- 'ZetadbApplication', 'ZetadbZptInsert', 'I18NLayer', 'ZetadbZptView', 'BrowserIdManager',
- 'ZetadbScriptSelectMaster', 'ZetadbSqlSelect', ]
-CLASSNAME_TO_SKIP = ['CatalogTool', 'MemberDataTool', 'SkinsTool', 'TypesTool',
- 'UndoTool', 'URLTool', 'WorkflowTool', 'DiscussionTool', 'MembershipTool',
- 'RegistrationTool', 'PropertiesTool', 'MetadataTool', 'SyndicationTool',
- 'PloneTool', 'NavigationTool', 'FactoryTool', 'FormTool', 'MigrationTool',
- 'CalendarTool', 'QuickInstallerTool', 'GroupsTool', 'GroupDataTool', 'MailHost',
- 'CookieCrumbler', 'ContentTypeRegistry', 'GroupUserFolder', 'CachingPolicyManager',
- 'InterfaceTool', 'PloneControlPanel', 'FormController', 'SiteErrorLog', 'SinTool',
- 'ArchetypeTool', 'RAMCacheManager', 'PloneArticleTool', 'SyndicationInformation',
- 'ActionIconsTool', 'AcceleratedHTTPCacheManager', 'ActionsTool', 'UIDCatalog',
- 'ReferenceCatalog', 'ContentPanelsTool', 'MimeTypesRegistry', 'LanguageTool',
- 'TransformTool']
-ID_TO_SKIP = ['Members', ]
+
+############## Move configuration to jsonmigrator.ini
+############## in DEFAULT section specify
+############## - CLASSNAME_TO_SKIP_LAUD (list separated by CARRIAGE_RETURN)
+############## - CLASSNAME_TO_SKIP (list separated by CARRIAGE_RETURN)
+
+def getconf(option, default):
+ global CONFIG
+ if not CONFIG.has_option('DEFAULT', option):
+ return default
+ else:
+ return CONFIG.get('DEFAULT', option)
+
+
+
+HOMEDIR = getconf('HOMEDIR',
+ '/Users/rok/Projects/yaco/unex_exported_data')
+logger.info("HOMEDIR : %s" % HOMEDIR)
+
+CLASSNAME_TO_SKIP_LAUD = [x.strip() for x \
+ in getconf('CLASSNAME_TO_SKIP_LAUD',
+ """ControllerPythonScript
+ ControllerPageTemplate
+ ControllerValidator
+ PythonScript
+ SQL
+ Connection
+ ZetadbScript
+ ExternalMethod
+ ZetadbSqlInsert
+ ZetadbMysqlda
+ SiteRoot
+ ZetadbApplication
+ ZetadbZptInsert
+ I18NLayer
+ ZetadbZptView
+ BrowserIdManager
+ ZetadbScriptSelectMaster
+ ZetadbSqlSelect""").splitlines()]
+
+CLASSNAME_TO_SKIP = [x.strip() for x \
+ in getconf('CLASSNAME_TO_SKIP',
+ """CatalogTool
+ MemberDataTool
+ SkinsTool
+ TypesTool
+ UndoTool
+ URLTool
+ WorkflowTool
+ DiscussionTool
+ MembershipTool
+ RegistrationTool
+ PropertiesTool
+ MetadataTool
+ SyndicationTool
+ PloneTool
+ NavigationTool
+ FactoryTool
+ FormTool
+ MigrationTool
+ CalendarTool
+ QuickInstallerTool
+ GroupsTool
+ GroupDataTool
+ MailHost
+ CookieCrumbler
+ ContentTypeRegistry
+ GroupUserFolder
+ CachingPolicyManager
+ InterfaceTool
+ PloneControlPanel
+ FormController
+ SiteErrorLog
+ SinTool
+ ArchetypeTool
+ RAMCacheManager
+ PloneArticleTool
+ SyndicationInformation
+ ActionIconsTool
+ AcceleratedHTTPCacheManager
+ ActionsTool
+ UIDCatalog
+ ReferenceCatalog
+ ContentPanelsTool
+ MimeTypesRegistry
+ LanguageTool
+ TransformTool""").splitlines()]
+
+ID_TO_SKIP = [x.strip() for x \
+ in getconf('ID_TO_SKIP',
+ """Members""").splitlines()]
+NON_FOLDERISH_CLASSNAME = [x.strip() for x \
+ in getconf('NON_FOLDERISH_CLASSNAME',
+ """PloneArticle""").splitlines()]
+JUST_TREAT_WAPPER = False
+try:
+ JUST_TREAT_WAPPER = eval(getconf('JUST_TREAT_WAPPER',False))
+except:
+ JUST_TREAT_WAPPER = False
+print 'ID_TO_SKIP %s ' % str(ID_TO_SKIP)
+
+try:
+ MAX_TREAT = int(getconf('MAX_TREAT', 0))
+except:
+ MAX_TREAT = 0
+
+try:
+ MAX_CACHE_DB = int(getconf('MAX_CACHE_DB', 500))
+except:
+ MAX_CACHE_DB = 500
def export_plone20(self):
@@ -44,7 +156,9 @@ def export_plone20(self):
COUNTER = 1
TODAY = datetime.today()
- TMPDIR = HOMEDIR+'/content_'+self.getId()+'_'+TODAY.strftime('%Y-%m-%d-%H-%M-%S')
+ TMPDIR = os.path.join(HOMEDIR,'content_%s_%s' % \
+ (self.getId(),
+ TODAY.strftime('%Y-%m-%d-%H-%M-%S')))
id_to_skip = self.REQUEST.get('id_to_skip', None)
if id_to_skip is not None:
@@ -54,7 +168,7 @@ def export_plone20(self):
shutil.rmtree(TMPDIR)
else:
os.mkdir(TMPDIR)
-
+
write(walk(self))
# TODO: we should return something more useful
@@ -62,39 +176,57 @@ def export_plone20(self):
def walk(folder):
+ global COUNTER
for item_id in folder.objectIds():
item = folder[item_id]
if item.__class__.__name__ in CLASSNAME_TO_SKIP or \
- item.getId() in ID_TO_SKIP:
+ item.getId() in ID_TO_SKIP or (JUST_TREAT_WAPPER and \
+ item.__class__.__name__\
+ not in CLASSNAME_TO_WAPPER_MAP) or \
+ (item.__class__.__name__ in CLASSNAME_TO_SKIP_LAUD):
+ logger.info('>> SKIPPING :: ['+item.__class__.__name__+'] '\
+ + item.absolute_url())
continue
- if item.__class__.__name__ in CLASSNAME_TO_SKIP_LAUD:
- print '>> SKIPPING :: ['+item.__class__.__name__+'] '+item.absolute_url()
+ if MAX_TREAT != 0 and COUNTER >= MAX_TREAT:
continue
+ logger.info('>> TREAT :: ('+ str(COUNTER) +')['+item.__class__.__name__+'] '\
+ + item.absolute_url())
yield item
if getattr(item, 'objectIds', None) and \
- item.objectIds():
+ item.objectIds() and \
+ item.__class__.__name__ not in NON_FOLDERISH_CLASSNAME:
for subitem in walk(item):
yield subitem
def write(items):
+
global COUNTER
for item in items:
- if item.__class__.__name__ not in CLASSNAME_TO_WAPPER_MAP.keys():
+ if item.__class__.__name__\
+ not in CLASSNAME_TO_WAPPER_MAP.keys():
import pdb; pdb.set_trace()
raise Exception, 'No wrapper defined for "'+item.__class__.__name__+ \
'" ('+item.absolute_url()+').'
try:
+
dictionary = CLASSNAME_TO_WAPPER_MAP[item.__class__.__name__](item)
write_to_jsonfile(dictionary)
COUNTER += 1
+ if (COUNTER % MAX_CACHE_DB)==0:
+ logger.info('Purge ZODB cache')
+ [item.Control_Panel.Database[x]._getDB().cacheMinimize() \
+ for x in item.Control_Panel.Database.getDatabaseNames()]
except:
- import pdb; pdb.set_trace()
+ print "there is an error on %s" % item.absolute_url()
+ #import pdb;pdb.set_trace();
+ raise
def write_to_jsonfile(item):
global COUNTER
+
SUB_TMPDIR = os.path.join(TMPDIR, str(COUNTER/1000)) # 1000 files per folder, so we dont reach some fs limit
if not os.path.isdir(SUB_TMPDIR):
@@ -106,33 +238,47 @@ def write_to_jsonfile(item):
for datafield in item['__datafields__']:
datafield_filepath = os.path.join(SUB_TMPDIR, str(COUNTER)+'.json-file-'+str(datafield_counter))
f = open(datafield_filepath, 'wb')
- f.write(item[datafield])
- item[datafield] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
+ if type(item[datafield]) is dict:
+ f.write(item[datafield]['data'])
+ del item[datafield]['data']
+ else:
+ f.write(item[datafield])
+ item[datafield] = {}
+ #f.write(item[datafield])
+ item[datafield]['path'] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
+ #item[datafield] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
f.close()
datafield_counter += 1
item.pop(u'__datafields__')
if '_plonearticle_attachments' in item:
for item2 in item['_plonearticle_attachments']:
+ if not item2.has_key('attachedFile'):
+ continue
datafield_filepath = os.path.join(SUB_TMPDIR, str(COUNTER)+'.json-file-'+str(datafield_counter))
- f = open(datafield_filepath, 'wb')
- f.write(item2['attachedFile'][0])
- item2['attachedFile'][0] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
+ f = open(datafield_filepath, 'wb')
+ f.write(item2['attachedFile'][0]['data'])
+ item2['attachedFile'][0]['data'] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
f.close()
datafield_counter += 1
if '_plonearticle_images' in item:
for item2 in item['_plonearticle_images']:
+ if not item2.has_key('attachedImage'):
+ continue
datafield_filepath = os.path.join(SUB_TMPDIR, str(COUNTER)+'.json-file-'+str(datafield_counter))
f = open(datafield_filepath, 'wb')
try:
- f.write(item2['attachedImage'][0])
+ f.write(item2['attachedImage'][0]['data'])
except:
import pdb; pdb.set_trace()
- item2['attachedImage'][0] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
+ item2['attachedImage'][0]['data'] = os.path.join(str(COUNTER/1000), str(COUNTER)+'.json-file-'+str(datafield_counter))
f.close()
datafield_counter += 1
-
+
f = open(os.path.join(SUB_TMPDIR, str(COUNTER)+'.json'), 'wb')
- simplejson.dump(item, f, indent=4)
+ try:
+ simplejson.dump(item, f, indent=4)
+ except:
+ raise str(item)
f.close()
@@ -142,6 +288,16 @@ def getPermissionMapping(acperm):
result[entry[0]] = entry[1]
return result
+def safe_decode(s, charset, errors):
+ if type(s) is type(u''):
+ return s
+ if hasattr(s, 'decode'):
+ return s.decode(charset, errors)
+
+ if s.__class__.__name__ == 'BaseUnit':
+ return str(s).decode(charset, errors)
+ else:
+ return s
class BaseWrapper(dict):
"""Wraps the dublin core metadata and pass it as tranmogrifier friendly style
@@ -151,30 +307,39 @@ def __init__(self, obj):
self.obj = obj
self.portal = getToolByName(obj, 'portal_url').getPortalObject()
+ relative_url = getToolByName(obj, 'portal_url').getRelativeContentURL
self.portal_utils = getToolByName(obj, 'plone_utils')
self.charset = self.portal.portal_properties.site_properties.default_charset
if not self.charset: # newer seen it missing ... but users can change it
self.charset = 'utf-8'
-
+
self['__datafields__'] = []
- self['_path'] = '/'.join(self.obj.getPhysicalPath())
-
+ #self['_path'] = '/'.join(self.obj.getPhysicalPath())
+ self['_path'] = relative_url(self.obj)
self['_type'] = self.obj.__class__.__name__
self['id'] = obj.getId()
- self['title'] = obj.title.decode(self.charset, 'ignore')
- self['description'] = obj.description.decode(self.charset, 'ignore')
- self['language'] = obj.language
- self['rights'] = obj.rights.decode(self.charset, 'ignore')
+ self['title'] = safe_decode(obj.title,self.charset, 'ignore')
+ self['description'] = safe_decode(obj.description,self.charset, 'ignore')
+ self['language'] = obj.Language()
+ self['rights'] = safe_decode(obj.rights,self.charset, 'ignore')
# for DC attrs that are tuples
for attr in ('subject', 'contributors'):
self[attr] = []
val_tuple = getattr(obj, attr, False)
if val_tuple:
for val in val_tuple:
- self[attr].append(val.decode(self.charset, 'ignore'))
+ self[attr].append(safe_decode(val,self.charset, 'ignore'))
self[attr] = tuple(self[attr])
+ # Creators
+ self['creators'] = []
+ val_tuple = obj.Creators()
+ if val_tuple:
+ for val in val_tuple:
+ self['creators'].append(safe_decode(val,self.charset, 'ignore'))
+
+
# for DC attrs that are DateTimes
datetimes_dict = {'creation_date': 'creation_date',
'modification_date': 'modification_date',
@@ -192,16 +357,28 @@ def __init__(self, obj):
for w in workflow_history:
for i, w2 in enumerate(workflow_history[w]):
workflow_history[w][i]['time'] = str(workflow_history[w][i]['time'])
- workflow_history[w][i]['comments'] = workflow_history[w][i]['comments'].decode(self.charset, 'ignore')
+ workflow_history[w][i]['comments'] = safe_decode(workflow_history[w][i]['comments'],self.charset, 'ignore')
except:
import pdb; pdb.set_trace()
self['_workflow_history'] = workflow_history
# default view
- _browser = '/'.join(self.portal_utils.browserDefault(aq_base(obj))[1])
- if _browser not in ['folder_listing']:
- self['_layout'] = ''
- self['_defaultpage'] = _browser
+
+ if 'layout' in obj.__dict__:
+ self['_layout'] = obj.__dict__['layout']
+ try:
+ _browser = self.plone_utils.browserDefault(aq_base(obj))[1]
+ except:
+ _browser = None
+ if _browser:
+ ## _browser can be value [None]
+ try:
+ _browser = '/'.join(_browser)
+ except:
+ _browser = ''
+ if _browser not in ['folder_listing']:
+ self['_layout'] = ''
+ self['_defaultpage'] = _browser
#elif obj.getId() != 'index_html':
# self['_layout'] = _browser
# self['_defaultpage'] = ''
@@ -219,7 +396,7 @@ def __init__(self, obj):
if typ == 'string':
if getattr(val, 'decode', False):
try:
- val = val.decode(self.charset, 'ignore')
+ val = safe_decode(val,self.charset, 'ignore')
except UnicodeEncodeError:
val = unicode(val)
else:
@@ -233,6 +410,8 @@ def __init__(self, obj):
for key, val in obj.__ac_local_roles__.items():
if key is not None:
self['_ac_local_roles'][key] = val
+ if 'Owner' in val:
+ self['_owner'] = key
self['_userdefined_roles'] = ()
if getattr(aq_base(obj), 'userdefined_roles', False):
@@ -256,7 +435,13 @@ def __init__(self, obj):
self['_permission_mapping'][perm['name']] = \
{'acquire': not unchecked,
'roles': new_roles}
-
+
+ if getattr(aq_base(obj), 'isCanonical', False):
+ if not obj.isCanonical():
+ canonical = obj.getCanonical()
+ self['_canonical'] = relative_url(canonical)
+
+
# self['_ac_inherited_permissions'] = {}
# if getattr(aq_base(obj), 'ac_inherited_permissions', False):
# oldmap = getPermissionMapping(obj.ac_inherited_permissions(1))
@@ -264,14 +449,16 @@ def __init__(self, obj):
# old_p = Permission(key, values, obj)
# self['_ac_inherited_permissions'][key] = old_p.getRoles()
- if getattr(aq_base(obj), 'getWrappedOwner', False):
- self['_owner'] = (1, obj.getWrappedOwner().getId())
- else:
+
+
+ #if getattr(aq_base(obj), 'getWrappedOwner', False):
+ # self['_owner'] = (1, obj.getWrappedOwner().getId())
+ #else:
# fallback
# not very nice but at least it works
# trying to get/set the owner via getOwner(), changeOwnership(...)
# did not work, at least not with plone 1.x, at 1.0.1, zope 2.6.2
- self['_owner'] = (0, obj.getOwner(info = 1).getId())
+ # self['_owner'] = (0, obj.getOwner(info = 1).getId())
def decode(self, s, encodings=('utf8', 'latin1', 'ascii')):
if self.charset:
@@ -281,14 +468,15 @@ def decode(self, s, encodings=('utf8', 'latin1', 'ascii')):
return s.decode(encoding)
except UnicodeDecodeError:
pass
- return s.decode(test_encodings[0], 'ignore')
+ return safe_decode(s,test_encodings[0], 'ignore')
class DocumentWrapper(BaseWrapper):
def __init__(self, obj):
super(DocumentWrapper, self).__init__(obj)
- self['text'] = obj.text.decode(self.charset, 'ignore')
+ if hasattr(obj, 'text'):
+ self['text'] = safe_decode(obj.text,self.charset, 'ignore')
class I18NFolderWrapper(BaseWrapper):
@@ -299,10 +487,10 @@ def __init__(self, obj):
lang = obj.getDefaultLanguage()
data = obj.folder_languages.get(lang, None)
if data is not None:
- self['title'] = data['title'].decode(self.charset, 'ignore')
- self['description'] = data['description'].decode(self.charset, 'ignore')
+ self['title'] = safe_decode(data['title'],self.charset, 'ignore')
+ self['description'] = safe_decode(data['description'],self.charset, 'ignore')
else:
- print 'ERROR: Cannot get default data for I18NFolder "%s"' % self['_path']
+ logger.error('ERROR: Cannot get default data for I18NFolder "%s"' % self['_path'])
# delete empty title in properties
for prop in self['_properties']:
@@ -316,7 +504,7 @@ def __init__(self, obj):
data = obj.folder_languages[lang]
for field in data:
self['_properties'].append(['%s_%s' % (lang, field),
- data[field].decode(self.charset, 'ignore'),
+ safe_decode(data[field],self.charset, 'ignore'),
'text'])
@@ -324,7 +512,7 @@ class LinkWrapper(BaseWrapper):
def __init__(self, obj):
super(LinkWrapper, self).__init__(obj)
- self['remoteUrl'] = obj.remote_url
+ self['remoteUrl'] = obj.remote_url()
class NewsItemWrapper(DocumentWrapper):
@@ -370,24 +558,25 @@ def __init__(self, obj):
class FileWrapper(BaseWrapper):
-
+ ## fs file ##
def __init__(self, obj):
super(FileWrapper, self).__init__(obj)
self['__datafields__'].append('_datafield_file')
data = str(obj.data)
if len(data) != obj.getSize():
- raise Exception, 'Problem while extracting data for File content type at '+obj.absolute_url()
+ raise Exception, 'Problem while extracting data for File content type at '+obj.absolute_url()
self['_datafield_file'] = data
-class ImageWrapper(BaseWrapper):
+class ImageWrapper(BaseWrapper):
+ ## fs image ##
def __init__(self, obj):
super(ImageWrapper, self).__init__(obj)
self['__datafields__'].append('_datafield_image')
data = str(obj.data)
if len(data) != obj.getSize():
- raise Exception, 'Problem while extracting data for Image content type at '+obj.absolute_url()
+ raise Exception, 'Problem while extracting data for Image content type at '+obj.absolute_url()
self['_datafield_image'] = data
@@ -397,18 +586,19 @@ def __init__(self, obj):
super(EventWrapper, self).__init__(obj)
self['startDate'] = str(obj.start_date)
self['endDate'] = str(obj.end_date)
- self['location'] = obj.location.decode(self.charset, 'ignore')
- self['contactName'] = obj.contact_name.decode(self.charset, 'ignore')
- self['contactEmail'] = obj.contact_email
- self['contactPhone'] = obj.contact_phone
- self['eventUrl'] = obj.event_url
+ self['location'] = safe_decode(obj.location,self.charset, 'ignore')
+ self['contactName'] = safe_decode(obj.contact_name(),self.charset, 'ignore')
+ self['contactEmail'] = obj.contact_email()
+ self['contactPhone'] = obj.contact_phone()
+ self['eventUrl'] = obj.event_url()
class ArchetypesWrapper(BaseWrapper):
def __init__(self, obj):
+
super(ArchetypesWrapper, self).__init__(obj)
-
+
fields = obj.schema.fields()
for field in fields:
type_ = field.__class__.__name__
@@ -443,22 +633,31 @@ def __init__(self, obj):
self[unicode(field.__name__)] = ['/'+i.absolute_url() for i in value]
else:
self[unicode(field.__name__)] = value.absolute_url()
- elif type_ in ['ImageField', 'FileField']:
+ elif type_ in ['ImageField', 'FileField', 'AttachmentField']:
+ #import pdb;pdb.set_trace();
fieldname = unicode('_data_'+field.__name__)
value = field.get(obj)
value2 = value
if type(value) is not str:
- value = str(value.data)
+ try:
+ value = str(value.data)
+ except:
+ import pdb;pdb.set_trace();
if value:
- size = value2.getSize()
+
self['__datafields__'].append(fieldname)
- self[fieldname] = {
- 'data': value,
- 'size': size, }
+ self[fieldname] = {}
+ for x in field.get(obj).__dict__:
+ if type(field.get(obj).__dict__[x]) in (int,str):
+ self[fieldname][x] = field.get(obj).__dict__[x]
+ self[fieldname]['data'] = value
+
elif type_ in ['ComputedField']:
pass
+
else:
- raise 'Unknown field type for ArchetypesWrapper.'
+
+ raise 'Unknown field type for ArchetypesWrapper : %s' % type_
def _guessFilename(self, data, fname='', mimetype='', default=''):
"""
@@ -487,12 +686,12 @@ def __init__(self, obj):
super(I18NLayerWrapper, self).__init__(obj)
lang = obj.portal_properties.site_properties.default_language
if lang not in obj.objectIds():
- print 'ERROR: Cannot get default data for I18NLayer "%s"' % self['_path']
+ logger.error('ERROR: Cannot get default data for I18NLayer "%s"' % self['_path'])
else:
real = obj[lang]
- self['title'] = real.title.decode(self.charset, 'ignore')
- self['description'] = real.description.decode(self.charset, 'ignore')
- self['text'] = real.text.decode(self.charset, 'ignore')
+ self['title'] = safe_decode(real.title,self.charset, 'ignore')
+ self['description'] = safe_decode(real.description,self.charset, 'ignore')
+ self['text'] = safe_decode(real.text,self.charset, 'ignore')
# Not lose information: generate properites es_title, en_title, etc.
# TODO: Export all archetypes, but I don't need now, only document important fields
@@ -502,13 +701,120 @@ def __init__(self, obj):
text = content.text)
for field in data:
self['_properties'].append(['%s_%s' % (lang, field),
- data[field].decode(self.charset, 'ignore'),
+ safe_decode(data[field],self.charset, 'ignore'),
'text'])
+def generateUniqueId(type_name=None):
+ """
+ Generate an id for the content
+ This is not the archetype's uid.
+ """
+ from DateTime import DateTime
+ from random import random
+
+ now = DateTime()
+ time = '%s.%s' % (now.strftime('%Y-%m-%d'), str(now.millis())[7:])
+ rand = str(random())[2:6]
+ prefix = ''
+ suffix = ''
+
+ if type_name is not None:
+ prefix = type_name.replace(' ', '_') + '.'
+ prefix = prefix.lower()
+
+ return prefix + time + rand + suffix
+
+
+def getNewModelName(model):
+ re_match = PAV3_MODEL_RE.search(model)
+ if re_match is not None:
+ model = 'pa_model%s' % (re_match.group(1) or '1',)
+ elif model == 'plonearticle_view':
+ model = 'pa_model1'
+ return model
+
+
+class Article322Wrapper(NewsItemWrapper):
+
+ def __init__(self, obj):
+ super(Article322Wrapper, self).__init__(obj)
+
+ #(Pdb) self.__ordered_attachment_refs__.getItems()
+ #['4e952a8c3af4b1bcedf38d475ac6049d']
+ d = {'__ordered_attachment_refs__' : ('_plonearticle_attachments',
+ 'FileProxy',
+ 'attachedFile',
+ 'getFile'),
+ '__ordered_image_refs__' : ('_plonearticle_images',
+ 'ImageProxy',
+ 'attachedImage',
+ 'getImage'),
+
+ '__ordered_link_refs__' : ('_plonearticle_refs',
+ 'LinkProxy',
+ 'attachedLink',
+ 'getRemoteUrl')}
+ ## layout
+
+ model = obj.getModel()
+ self['_layout'] = getNewModelName(model)
+
+
+ ids = obj.objectIds()
+ for x in d:
+ slot_name = d[x][0]
+ id_name = d[x][1]
+ field_name = d[x][2]
+ accessor = d[x][3]
+ self[slot_name] = []
+ for refid in getattr(obj,x).getItems():
+ ref = None
+ try:
+ ref = getattr(obj.at_references, refid).getTargetObject()
+ except:
+ ## ghost ref
+ logger.exception("Attribut rror during migration on %s"\
+ % str(obj))
+ continue ## just ignore it...
+ inner = {
+ 'id': (generateUniqueId(id_name), {}),
+ 'title': (safe_decode(ref.Title(),
+ self.charset, 'ignore'), {}),
+ 'description': (safe_decode(ref.Description(),
+ self.charset,
+ 'ignore'), {}),}
+ if ref.id in ids:
+ ### internal
+ innerfile = getattr(ref, accessor)()
+ if innerfile:
+ di = {}
+ try:
+ data = str(innerfile.data)
+ for x in innerfile.__dict__:
+ if type(innerfile.__dict__[x]) in (int,str):
+ di[x] = innerfile.__dict__[x]
+ except:
+ data = innerfile
+
+
+ di['data'] = data
+ inner[field_name] = (di, {})
+
+ else:
+ #### external
+ inner['referencedContent'] = (ref.UID(), {})
+ self[slot_name].append(inner)
+
+
+
+
+
+
class ArticleWrapper(NewsItemWrapper):
def __init__(self, obj):
+
super(ArticleWrapper, self).__init__(obj)
try:
self['cooked_text'] = obj.cooked_text.decode(self.charset)
@@ -520,8 +826,8 @@ def __init__(self, obj):
item = obj[item_id]
plonearticle_attachments.append({
'id': (item_id, {}),
- 'title': (item.title.decode(self.charset, 'ignore'), {}),
- 'description': (item.description.decode(self.charset, 'ignore'), {}),
+ 'title': (safe_decode(item.title, self.charset, 'ignore'), {}),
+ 'description': (safe_decode(item.description, self.charset, 'ignore'), {}),
'attachedFile': [item.getFile(), {}],
})
self['_plonearticle_attachments'] = plonearticle_attachments
@@ -531,8 +837,8 @@ def __init__(self, obj):
item = obj[item_id]
plonearticle_images.append({
'id': (item_id, {}),
- 'title': (item.title.decode(self.charset, 'ignore'), {}),
- 'description': (item.description.decode(self.charset, 'ignore'), {}),
+ 'title': (safe_decode(item.title, self.charset, 'ignore'), {}),
+ 'description': (safe_decode(self.charset, 'ignore'), {}),
'attachedImage': [str(item.data), {}],
})
self['_plonearticle_images'] = plonearticle_images
@@ -624,34 +930,48 @@ def __init__(self, obj):
# self['__datafields__'].append('document_src')
# TODO: should be also possible to set it with through parameters
-CLASSNAME_TO_WAPPER_MAP = {
- 'LargePloneFolder': BaseWrapper,
- 'Folder': BaseWrapper,
- 'PloneSite': BaseWrapper,
- 'PloneFolder': BaseWrapper,
- 'Document': DocumentWrapper,
- 'File': FileWrapper,
- 'Image': ImageWrapper,
- 'Link': LinkWrapper,
- 'Event': EventWrapper,
- 'NewsItem': NewsItemWrapper,
- 'Favorite': LinkWrapper,
- 'Topic': BaseWrapper,
- 'ListCriterion': ListCriteriaWrapper,
- 'SimpleStringCriterion': StringCriteriaWrapper,
- 'SortCriterion': SortCriteriaWrapper,
- 'FriendlyDateCriterion': DateCriteriaWrapper,
-
- # custom ones
- 'I18NFolder': I18NFolderWrapper,
- 'I18NLayer': I18NLayerWrapper,
- 'PloneArticle': ArticleWrapper,
- 'ZPhotoSlides': ZPhotoSlidesWrapper,
- 'ZPhoto': ZPhotoWrapper,
- 'PloneLocalFolderNG': ArchetypesWrapper,
- 'LocalFS': LocalFSWrapper,
- 'ContentPanels': ContentPanels,
- 'DTMLMethod': ZopeObjectWrapper,
- 'ZopePageTemplate': ZopeObjectWrapper,
-
-}
+CLASSNAME_TO_WAPPER_MAP = {}
+if CONFIG.has_section('CLASSNAME_TO_WAPPER_MAP'):
+ for x in CONFIG.items('CLASSNAME_TO_WAPPER_MAP'):
+
+ try:
+ CLASSNAME_TO_WAPPER_MAP[x[0]] = eval(x[1].strip())
+ logger.debug("map %s to %s" % (x[0], x[1]) )
+ except:
+ logger.info("cant add class for mapping %s" % x[0])
+ pass
+else:
+ print "load default CLASSNAME_TO_WAPPER_MAP"
+ CLASSNAME_TO_WAPPER_MAP = {
+ 'LargePloneFolder': BaseWrapper,
+ 'Folder': BaseWrapper,
+ 'PloneSite': BaseWrapper,
+ 'PloneFolder': BaseWrapper,
+ 'Document': DocumentWrapper,
+ 'File': FileWrapper,
+ 'Image': ImageWrapper,
+ 'Link': LinkWrapper,
+ 'Event': EventWrapper,
+ 'NewsItem': NewsItemWrapper,
+ 'Favorite': LinkWrapper,
+ 'Topic': BaseWrapper,
+ 'ListCriterion': ListCriteriaWrapper,
+ 'SimpleStringCriterion': StringCriteriaWrapper,
+ 'SortCriterion': SortCriteriaWrapper,
+ 'FriendlyDateCriterion': DateCriteriaWrapper,
+
+ # custom ones
+ 'I18NFolder': I18NFolderWrapper,
+ 'I18NLayer': I18NLayerWrapper,
+ 'PloneArticle': ArticleWrapper,
+ 'ZPhotoSlides': ZPhotoSlidesWrapper,
+ 'ZPhoto': ZPhotoWrapper,
+ 'PloneLocalFolderNG': ArchetypesWrapper,
+ 'LocalFS': LocalFSWrapper,
+ 'ContentPanels': ContentPanels,
+ 'DTMLMethod': ZopeObjectWrapper,
+ 'ZopePageTemplate': ZopeObjectWrapper,
+
+ }
+
+
diff --git a/setup.py b/setup.py
index 307e19e..18b3504 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
from setuptools import setup, find_packages
import os
-version = '0.1.1'
+version = '0.2.6dev'
setup(name='collective.blueprint.jsonmigrator',
version=version,
@@ -13,8 +13,8 @@
"Programming Language :: Python",
],
keywords='',
- author='',
- author_email='',
+ author='garbas + yboussard',
+ author_email='y.boussard@alterway.fr',
url='http://svn.plone.org/svn/collective/',
license='GPL',
packages=find_packages(exclude=['ez_setup']),