* Finalized user and skin conversions

* Fixed output encoding
This commit is contained in:
Tobi Schäfer 2007-10-18 02:15:10 +00:00
parent 00a6aa86a1
commit 2578fcd36f
5 changed files with 152 additions and 335 deletions

View file

@ -100,41 +100,41 @@ convert.sites = function() {
convert.content = function() {
convert.xml("content");
convert.tags("content");
};
convert.users = function(type) {
var query = "select user_id, hash, salt, user_url from av_user";
rows = db.executeRetrieval(query);
while (rows && rows.next()) {
sql = "update av_user set metadata = " + quote({
hash: rows.getColumnItem("hash"),
salt: rows.getColumnItem("salt"),
url: rows.getColumnItem("user_url")
}.toSource()) + " where user_id = " + rows.getColumnItem("user_id");
writeln(sql);
db.executeCommand(sql);
}
convert.users = function() {
retrieve("select id, hash, salt, USER_URL from AV_USER");
traverse(function() {
var metadata = {
hash: this.hash,
salt: this.salt,
url: this.USER_URL
}
execute("update AV_USER set metadata = " + quote(metadata.toSource()) +
" where id = " + this.id);
});
}
convert.xml = function(table) {
var metadata = function(xml) {
var clean = xml.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
var object = Xml.readFromString(clean);
return object.toSource().replace(/\\/g, "\\\\").replace(/'/g, "\\'");
try {
return Xml.readFromString(clean).toSource();
} catch (ex) {
app.debug(xml);
}
return {}.toSource();
};
retrieve(sql("jsonize", table));
traverse(function() {
try {
if (!this.xml) {
return;
}
var data = metadata(this.xml);
execute("update " + table + " set metadata = " +
quote(data) + " where id = " + this.id);
} catch (ex) {
println(ex);
if (!this.xml) {
return;
}
var data = metadata(this.xml);
execute("update " + table + " set metadata = " +
quote(data) + " where id = " + this.id);
});
}
@ -167,273 +167,30 @@ convert.tags = function(table) {
}
convert.skins = function() {
// FIXME
}
Root.prototype.xml2json_action = function() {
var db = getDBConnection("antville");
var table, chunkSize = 10000;
var getSql = function(type) {
var sql;
var column = (table === "TEXT" ? "CONTENT" : "PREFERENCES");
switch (type) {
case "create":
sql = "alter table AV_$T add $T_$C_NEW mediumtext default " +
"null after $T_$C";
break;
case "max":
//sql = "select 1 as max";
sql = "select max($T_ID) as max, count($T_ID) as amount from AV_$T";
break;
case "list":
//sql = "select $T_ID, $T_$C from AV_$T where TEXT_F_SITE = 3" +
// " order by $T_ID limit 1000";
sql = "select $T_ID, $T_$C as old, $T_$C_NEW as new from AV_$T " +
"where $T_ID > " + arguments[1] + " order by $T_ID limit " +
chunkSize;
break;
case "update":
sql = "update AV_$T set $T_$C_NEW = '" + arguments[2] +
"' where $T_ID = " + arguments[1];
break;
case "rename":
sql = "alter table AV_$T change $T_$C $T_$C_OLD " +
"mediumtext default null; alter table AV_$T change " +
"$T_$C_NEW $T_$C mediumtext default null";
break;
case "drop":
sql = "drop column $T_PREFERENCES from AV_$T";
break;
default:
throw Error("No such SQL");
}
sql = sql.replace(/\$T/g, table).replace(/\$C/g, column);
//(type !== "update") && writeln(sql);
return sql;
};
app.invokeAsync(null, function() {
var log = new java.io.FileWriter("apps/antville/conversion.log");
var rows, max, amount, data, counter, ref;
var id = 0;
main:
for each (table in ["TEXT"]) {
//for each (table in ["TEXT", "SITE", "LAYOUT"]) {
db.executeCommand(getSql("create"));
rows = db.executeRetrieval(getSql("max"));
rows.next();
max = rows.getColumnItem("max");
amount = rows.getColumnItem("amount");
rows.release();
counter = 0;
while (id < max) {
rows = db.executeRetrieval(getSql("list", id));
while (rows && rows.next()) {
id = rows.getColumnItem(table + "_ID");
if (rows.getColumnItem("new")) {
continue;
} else {
app.debug(id + " has no new data, yet!");
continue;
}
try {
data = rows.getColumnItem("old");
data = data.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
data = Xml.readFromString(data);
db.executeCommand(getSql("update", id,
data.toSource().replace(/\\/g,
"\\\\").replace(/'/g, "\\'")));
} catch (x) {
ref = Story.getById(id) || Comment.getById(id);
log.write(ref.href() + "\n");
log.write(x + "\n");
log.write("***\n");
log.flush();
//app.logger.error("!!! Error in XML data of TEXT_ID " + id);
//app.logger.error(x);
//break main;
}
}
rows && rows.release();
counter += chunkSize;
app.logger.info("Converted " + table + " " + counter + " of " +
amount + " records");
app.logger.info("Current ID: " + id + " of " + max);
}
//db.executeCommand(getSql("rename"));
//db.executeCommand(getSql("drop"));
}
log.write("### Finished conversion with ID " + id);
log.close();
}, [], -1);
};
Root.prototype.topic2tag_action = function() {
return;
var db = getDBConnection("antville");
var tagCounter = 1;
var hubCounter = 1;
function quote(s) {
return "'" + s + "'";
}
function createTags(siteId, taggedType) {
var sql;
var query = "select text_topic as tag from av_text where text_f_site = " +
siteId + " and text_topic is not null group by text_topic";
if (taggedType === "Image") {
query = query.replace(/text/g, "image");
}
writeln(query);
var rows = db.executeRetrieval(query);
while (rows && rows.next()) {
sql = "insert into tag values (" + [tagCounter, siteId,
quote(rows.getColumnItem("tag").replace(/^[\/\.]*$/, "?")),
quote(taggedType)] + ")";
writeln(sql);
db.executeCommand(sql);
tagCounter += 1;
}
rows && rows.release();
return;
}
function createHubs(siteId, taggedType) {
var sql;
var query = "select text_topic as name, tag.id as tagId, " +
"text_content_new as meta, text_id as taggedId, " +
"text_f_user_modifier as modifier, text_f_user_creator as " +
"creator, text_f_site as siteId from av_text, tag where " +
"text_topic is not null and text_topic = tag.name and " +
"tag.type = '" + taggedType + "' and text_f_site = " + siteId;
if (taggedType === "Image") {
query = query.replace(/text/g, "image");
query = query.replace(/content_new/g, "metadata");
}
writeln(query);
var rows = db.executeRetrieval(query);
while (rows && rows.next()) {
sql = "insert into tag_hub values (" + [hubCounter,
rows.getColumnItem("tagId"), rows.getColumnItem("taggedId"),
quote(taggedType), rows.getColumnItem("modifier") ||
rows.getColumnItem("creator")] + ")";
writeln(sql);
db.executeCommand(sql);
data = rows.getColumnItem("meta");
if (data) {
data = eval(data);
data.tags = rows.getColumnItem("name");
} else {
data = {tags: rows.getColumnItem("name")};
}
data = data.toSource().replace(/\\/g, "\\\\").replace(/'/g, "\\'");
sql = "update av_text set text_content_new = '" + data +
"' where text_id = " + rows.getColumnItem("taggedId");
if (taggedType === "Image") {
sql = sql.replace(/text/g, "image");
sql = sql.replace(/content_new/g, "metadata");
}
writeln(sql);
db.executeCommand(sql);
hubCounter += 1;
}
rows && rows.release();
return;
}
//var id = 1184; //3;
app.invokeAsync(null, function() {
/* createTags(id, "Story");
createHubs(id, "Story");
createTags(id, "Image");
createHubs(id, "Image");
return; */
var max = root.size();
for (var i=1; i<=max; i+=1) {
var site = root.get(i-1); // Site.getById(11);
app.logger.info("Tagging stories of site " + site._id + " ``" +
site.alias + "'' (" + i + " of " + max + ")");
createTags(site._id, "Story");
createHubs(site._id, "Story");
createTags(site._id, "Image");
createHubs(site._id, "Image");
continue;
var smax = site.stories.size();
counter = 0;
for (var n=0; n<smax; n+=1) {
var story = site.stories.get(n);
if (story.topic) {
//app.logger.info("***** Tagging story " + (n+1) + " of " + smax);
story.tags.add(new TagJoin(story.topic, story, site,
story.modifier || story.creator));
story.content.set("tags", story.topic);
res.commit();
}
}
}
}, [], -1);
};
Root.prototype.exportSkins_action = function() {
var db = getDBConnection("antville");
var exportSkins = function(query) {
var lastId;
var rows = db.executeRetrieval(query);
while (rows && rows.next()) {
var fpath = getProperty("staticPath") + rows.getColumnItem("SITE_ALIAS") +
"/layouts/" + rows.getColumnItem("LAYOUT_ALIAS") + "/" +
rows.getColumnItem("SKIN_PROTOTYPE");
var dump = function(sql) {
retrieve(sql);
traverse(function() {
var fpath = app.dir + "/../static/" + this.site_name +
"/layouts/" + this.name + "/" + this["prototype"];
var file = new java.io.File(fpath);
file.mkdirs();
file = new java.io.File(fpath,
rows.getColumnItem("SKIN_NAME").replace(/\//, "_") + ".skin");
//app.logger.info("Exporting skin #" + counter + ": " + file);
file = new java.io.File(file,
this.skin_name.replace(/\//, "_") + ".skin");
println(file);
file["delete"]();
var fos = new java.io.FileOutputStream(file);
var bos = new java.io.BufferedOutputStream(fos);
var writer = new java.io.OutputStreamWriter(bos, "UTF-8");
//var writer = new java.io.FileWriter(file);
writer.write(rows.getColumnItem("SKIN_SOURCE") || "");
writer.write(this.SKIN_SOURCE || "");
writer.close();
bos.close();
fos.close();
lastId = rows.getColumnItem("SKIN_ID");
}
rows && rows.release();
return lastId;
};
});
}
// Export skins of root layouts
exportSkins("select SKIN_ID, 'default' as SITE_ALIAS, LAYOUT_ALIAS, " +
"SKIN_PROTOTYPE, SKIN_NAME, SKIN_SOURCE from AV_LAYOUT, AV_SKIN " +
"where LAYOUT_F_SITE is null and SKIN_F_LAYOUT = LAYOUT_ID;");
// Export skins of site layouts
app.invokeAsync(null, function() {
var counter = 0;
var lastId = 0;
var rows = db.executeRetrieval("select max(SKIN_ID) as max, count(*) " +
"as amount from AV_SKIN");
rows.next()
var max = rows.getColumnItem("max");
var amount = rows.getColumnItem("amount");
rows.release();
while (lastId < max) {
lastId = exportSkins("select SKIN_ID, SITE_ALIAS, LAYOUT_ALIAS, " +
"SKIN_PROTOTYPE, SKIN_NAME, SKIN_SOURCE from AV_SITE, AV_LAYOUT, " +
"AV_SKIN where SKIN_F_LAYOUT = LAYOUT_ID and LAYOUT_F_SITE = " +
"SITE_ID and SKIN_ID > " + lastId + " order by SKIN_ID limit 100;");
counter += 100;
app.logger.info("Exported " + counter + " of " + amount + " skins");
app.logger.info("Current ID: " + lastId + " of " + max);
}
}, [], -1);
return;
};
dump(sql("skins"));
dump("select skin.id, 'www' as site_name, layout.name, " +
"skin.name as skin_name, prototype, SKIN_SOURCE from " +
"skin, layout where skin.layout_id = layout.id and " +
"layout.site_id is null");
}

View file

@ -28,7 +28,7 @@ var scheduler = setInterval(function() {
//-->
</script>
Antville Updater v0.1
<h3>Antville Updater v0.1</h3>
<pre id="out"></pre>
</body>

View file

@ -21,7 +21,7 @@ var ResultWrapper = function(result) {
var print = function(str) {
app.data.out || (app.data.out = new java.lang.StringBuffer());
app.data.out.append(str);
app.data.out.append(encodeForm(str));
return;
}

View file

@ -61,7 +61,7 @@ alter table AV_FILE change column FILE_F_USER_MODIFIER modifier_id mediumint(10)
alter table AV_FILE rename file;
#!files
alter table file drop column FILE_MIMETYPE;
alter table file drop column FILE_NAMEE;
alter table file drop column FILE_NAME;
alter table file drop column FILE_SIZE;
alter table file drop column FILE_DESCRIPTION;
@ -159,6 +159,7 @@ alter table AV_POLL change column POLL_QUESTION question mediumtext;
alter table AV_POLL change column POLL_CLOSETIME closed datetime;
alter table AV_POLL change column POLL_CREATETIME created datetime;
alter table AV_POLL change column POLL_MODIFYTIME modified datetime;
alter table AV_POLL drop column POLL_CLOSED;
alter table AV_POLL rename poll;
<% #AV_SITE %>
@ -237,48 +238,105 @@ alter table AV_SKIN change column SKIN_F_USER_MODIFIER modifier_id mediumint(10)
alter table AV_SKIN change column SKIN_CREATETIME created datetime;
alter table AV_SKIN change column SKIN_MODIFYTIME modified datetime;
alter table AV_SKIN rename skin;
#!skins
alter table skin drop column SKIN_ISCUSTOM;
alter table skin drop column SKIN_SOURCE;
<% #skins %>
select skin.id, site.name as site_name, layout.name, skin.name as skin_name,
prototype, SKIN_SOURCE from site, layout, skin where skin.layout_id =
layout.id and layout.site_id = site.id order by skin.id
<% #AV_SYSLOG %>
insert into log (context_type, context_id, created, creator_id, action) select 'User', user.id, SYSLOG_CREATETIME, SYSLOG_F_USER_CREATOR, SYSLOG_ENTRY from user, AV_SYSLOG where SYSLOG_OBJECT = user.name and SYSLOG_TYPE = 'user';
insert into log (context_type, context_id, created, creator_id, action) select 'Site', site.id, SYSLOG_CREATETIME, SYSLOG_F_USER_CREATOR, SYSLOG_ENTRY from site, AV_SYSLOG where SYSLOG_OBJECT = site.name and (SYSLOG_TYPE = 'site' or SYSLOG_TYPE = 'weblog');
insert into log (context_type, context_id, created, creator_id, action) select 'Root', 1, SYSLOG_CREATETIME, SYSLOG_F_USER_CREATOR, 'setup' from AV_SYSLOG where SYSLOG_TYPE = 'system';
<% #AV_TEXT %>
#alter table AV_TEXT change column TEXT_ID id mediumint(10);
#alter table AV_TEXT add column metadata mediumtext;
#alter table AV_TEXT change column TEXT_CONTENT xml mediumtext;
#alter table AV_TEXT rename content;
<% #AV_TEXT_ %>
alter table AV_TEXT change column TEXT_ID id mediumint(10);
alter table content change column TEXT_TOPIC topic varchar(255);
alter table AV_TEXT add column metadata mediumtext;
alter table AV_TEXT change column TEXT_CONTENT xml mediumtext;
alter table AV_TEXT rename content;
#!content
<% #_ %>
alter table av_text add column parent_type varchar(30);
update av_text set parent_type = "Story";
update av_text set parent_type = "Comment" where parent_id is not null;
alter table av_text add column status enum('closed','pending','readonly','public','shared','open');
update av_text set status = 'private';
update av_text set status = 'public' where text_isonline <> 0 or parent_type = "Comment";
update av_text set status = 'shared' where text_editableby = 1;
update av_text set status = 'open' where text_editableby = 2;
alter table av_text add column mode enum('hidden','featured');
update av_text set mode = 'hidden' where text_isonline = 1;
update av_text set mode = 'featured' where text_isonline = 2;
alter table av_text add column comment_mode enum('closed','readonly','moderated','open');
update av_text set comment_mode = 'closed';
update av_text set comment_mode = 'open' where text_hasdiscussions = 1;
alter table av_text change column text_alias name varchar(255);
alter table av_text change column text_f_site site_id mediumint(10);
alter table av_text change column text_f_text_story story_id mediumint(10);
alter table av_text change column text_f_text_parent parent_id mediumint(10);
alter table av_text change column text_reads requests mediumint(10);
alter table av_text change column text_prototype prototype enum('Story','Comment');
alter table av_text change column text_content_new metadata mediumtext;
alter table av_text change column text_f_user_creator creator_id mediumint(10);
alter table av_text change column text_f_user_modifier modifier_id mediumint(10);
alter table av_text change column text_createtime created datetime;
alter table av_text change column text_modifytime modified datetime;
alter table content add column parent_type varchar(30);
alter table content change column TEXT_ALIAS name varchar(255);
alter table content change column TEXT_F_SITE site_id mediumint(10);
alter table content change column TEXT_F_TEXT_STORY story_id mediumint(10);
alter table content change column TEXT_READS requests mediumint(10);
alter table content change column TEXT_F_USER_CREATOR creator_id mediumint(10);
alter table content change column TEXT_F_USER_MODIFIER modifier_id mediumint(10);
alter table content change column TEXT_CREATETIME created datetime;
alter table content change column TEXT_MODIFYTIME modified datetime;
alter table content change column TEXT_F_TEXT_PARENT parent_id mediumint(10);
alter table content change column TEXT_PROTOTYPE prototype enum('Story','Comment');
## Set parent mapping
update content set parent_type = "Story";
update content set parent_type = "Comment" where parent_id is not null;
update content set parent_id = story_id where parent_id is null and prototype = "Comment" and parent_type = "Story";
## Set status
alter table content add column status enum('closed','pending','readonly','public','shared','open');
update content set status = 'closed';
update content set status = 'public' where TEXT_ISONLINE <> 0 or parent_type = "Comment";
update content set status = 'shared' where TEXT_EDITABLEBY = 1;
update content set status = 'open' where TEXT_EDITABLEBY = 2;
## Set mode
alter table content add column mode enum('hidden','featured');
update content set mode = 'hidden' where TEXT_ISONLINE = 1;
update content set mode = 'featured' where TEXT_ISONLINE = 2;
## Set comment mode
alter table content add column comment_mode enum('closed','readonly','moderated','open');
update content set comment_mode = 'closed';
update content set comment_mode = 'open' where TEXT_HASDISCUSSIONS = 1;
## Remove obsolete columns
alter table content drop column xml;
alter table content drop column topic;
alter table content drop column TEXT_DAY;
alter table content drop column TEXT_TITLE;
alter table content drop column TEXT_TEXT;
alter table content drop column TEXT_RAWCONTENT;
alter table content drop column TEXT_ISONLINE;
alter table content drop column TEXT_EDITABLEBY;
alter table content drop column TEXT_HASDISCUSSIONS;
alter table content drop column TEXT_IPADDRESS;
<% #AV_USER %>
alter table AV_USER change column USER_ID id mediumint(10);
alter table AV_USER add column metadata mediumtext default NULL;
alter table AV_USER add column `status` enum('blocked','regular','trusted','privileged') not null;
update AV_USER set status = 'regular';
update AV_USER set status = 'blocked' where USER_ISBLOCKED = 1;
update AV_USER set status = 'trusted' where USER_ISTRUSTED = 1;
update AV_USER set status = 'privileged' where USER_ISSYSADMIN = 1;
alter table AV_USER add column hash varchar(32) default NULL;
alter table AV_USER add column salt varchar(12) default NULL;
update AV_USER set salt = conv(floor(0 + (rand() * pow(2, 48))), 10, 16), hash = md5(concat(user_password, salt));
#!users
## After conversion to Metadata these columns are obsolete
alter table AV_USER drop column hash;
alter table AV_USER drop column salt;
alter table AV_USER drop column USER_URL;
## Passwords are not stored in the database anymore
alter table AV_USER drop column USER_PASSWORD;
## Option to display e-mail in public is obsolete
alter table AV_USER drop column USER_EMAIL_ISPUBLIC;
## User status is now stored in one column
alter table AV_USER drop column USER_ISBLOCKED;
alter table AV_USER drop column USER_ISTRUSTED;
alter table AV_USER drop column USER_ISSYSADMIN;
## Renaming the remaining columns with nice names
alter table AV_USER change column USER_NAME name varchar(30);
alter table AV_USER change column USER_EMAIL email varchar(255);
alter table AV_USER change column USER_REGISTERED created datetime;
alter table AV_USER change column USER_LASTVISIT modified datetime;
alter table AV_USER rename user;
<% #AV_VOTE %>
alter table AV_VOTE change column VOTE_ID id mediumint(10);
alter table AV_VOTE change column VOTE_F_POLL poll_id mediumint(10);
alter table AV_VOTE change column VOTE_F_CHOICE choice_id mediumint(10);
alter table AV_VOTE change column VOTE_F_USER creator_id mediumint(10);
alter table AV_VOTE change column VOTE_USERNAME creator_name varchar(255);
alter table AV_VOTE change column VOTE_CREATETIME created datetime;
alter table AV_VOTE change column VOTE_MODIFYTIME modified datetime;
alter table AV_VOTE rename vote;

View file

@ -1,18 +1,20 @@
Root.prototype.start_action = function() {
app.invokeAsync(global, function() {
//execute(sql("tag"));
//execute(sql("tag_hub"));
//update("AV_ACCESSLOG");
//update("AV_CHOICE");
//update("AV_FILE");
//update("AV_IMAGE");
//update("AV_LAYOUT");
//update("AV_MEMBERSHIP");
//update("AV_POLL");
//update("AV_SITE");
//update("AV_SKIN");
//update("AV_SYSLOG");
execute(sql("tag"));
execute(sql("tag_hub"));
update("AV_ACCESSLOG");
update("AV_CHOICE");
update("AV_FILE");
update("AV_IMAGE");
update("AV_LAYOUT");
update("AV_MEMBERSHIP");
update("AV_POLL");
update("AV_SITE");
update("AV_SKIN");
update("AV_SYSLOG");
update("AV_TEXT");
update("AV_USER");
update("AV_VOTE");
return;
}, [], -1);
renderSkin("Global");