Loading into a target table
up vote
2
down vote
favorite
Today I tried Bonobo + SQLAlchemy (for the first time), so it's bound to be full of bad practices. However, this is working code. I'm hoping for some feedback to for doing it correctly.
I started out using the SQLAlchemy extension for Bonobo, but I ran into limitations, so I wrote my own loader. My DB is SQLite3, so it doesn't support multiple connections.
What this does is read from one table, do a simple transformation and load into a target table with a different table structure.
SQLite database:
CREATE TABLE sys_batch (
batch_id INTEGER PRIMARY KEY AUTOINCREMENT,
batch_name VARCHAR2 ( 100 ) );
CREATE TABLE sys_batch_tgt (
batch_id INTEGER PRIMARY KEY,
name_of_batch VARCHAR2 ( 100 ) );
Python code:
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.sql import select
from sqlalchemy.orm import Session
import bonobo
#import bonobo_sqlalchemy
engine = sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/coolstuff.db')
Base = automap_base()
Base.prepare(engine, reflect=True)
# Tables
sys_batch = Base.classes.sys_batch
sys_batch_tgt = Base.classes.sys_batch_tgt
#def get_services():
# return {'sqlalchemy.pgengine': sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/carauction.db')}
def extract_sys_batch():
conn = engine.connect()
s = select([sys_batch])
result = conn.execute(s)
#conn.close()
row = result.fetchone()
yield row
# convert incoming SQLAlchemy "ProxyRow" to a dict, so we can manipulate it.
def tfm_0(row):
tgt_row = {}
tgt_row['batch_id'] = row['batch_id']
tgt_row['name_of_batch'] = row['batch_name']
return tgt_row
def tfm_1(row):
row['batch_id'] = row['batch_id'] + 100
return row
def ldr_1(row):
# write to target
session = Session(engine)
thing = sys_batch_tgt(**row)
session.add(thing)
session.commit()
def get_graph(**options):
return bonobo.Graph(
extract_sys_batch,
tfm_0,
tfm_1,
ldr_1,
bonobo.PrettyPrinter(),
)
# The __main__ block actually execute the graph.
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
#each transformation is its own thread, so we have to open/close in there.
#conn = engine.connect()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**options)) #, services=get_services(**options)
#conn.close()
python python-3.x sqlalchemy
New contributor
add a comment |
up vote
2
down vote
favorite
Today I tried Bonobo + SQLAlchemy (for the first time), so it's bound to be full of bad practices. However, this is working code. I'm hoping for some feedback to for doing it correctly.
I started out using the SQLAlchemy extension for Bonobo, but I ran into limitations, so I wrote my own loader. My DB is SQLite3, so it doesn't support multiple connections.
What this does is read from one table, do a simple transformation and load into a target table with a different table structure.
SQLite database:
CREATE TABLE sys_batch (
batch_id INTEGER PRIMARY KEY AUTOINCREMENT,
batch_name VARCHAR2 ( 100 ) );
CREATE TABLE sys_batch_tgt (
batch_id INTEGER PRIMARY KEY,
name_of_batch VARCHAR2 ( 100 ) );
Python code:
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.sql import select
from sqlalchemy.orm import Session
import bonobo
#import bonobo_sqlalchemy
engine = sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/coolstuff.db')
Base = automap_base()
Base.prepare(engine, reflect=True)
# Tables
sys_batch = Base.classes.sys_batch
sys_batch_tgt = Base.classes.sys_batch_tgt
#def get_services():
# return {'sqlalchemy.pgengine': sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/carauction.db')}
def extract_sys_batch():
conn = engine.connect()
s = select([sys_batch])
result = conn.execute(s)
#conn.close()
row = result.fetchone()
yield row
# convert incoming SQLAlchemy "ProxyRow" to a dict, so we can manipulate it.
def tfm_0(row):
tgt_row = {}
tgt_row['batch_id'] = row['batch_id']
tgt_row['name_of_batch'] = row['batch_name']
return tgt_row
def tfm_1(row):
row['batch_id'] = row['batch_id'] + 100
return row
def ldr_1(row):
# write to target
session = Session(engine)
thing = sys_batch_tgt(**row)
session.add(thing)
session.commit()
def get_graph(**options):
return bonobo.Graph(
extract_sys_batch,
tfm_0,
tfm_1,
ldr_1,
bonobo.PrettyPrinter(),
)
# The __main__ block actually execute the graph.
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
#each transformation is its own thread, so we have to open/close in there.
#conn = engine.connect()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**options)) #, services=get_services(**options)
#conn.close()
python python-3.x sqlalchemy
New contributor
add a comment |
up vote
2
down vote
favorite
up vote
2
down vote
favorite
Today I tried Bonobo + SQLAlchemy (for the first time), so it's bound to be full of bad practices. However, this is working code. I'm hoping for some feedback to for doing it correctly.
I started out using the SQLAlchemy extension for Bonobo, but I ran into limitations, so I wrote my own loader. My DB is SQLite3, so it doesn't support multiple connections.
What this does is read from one table, do a simple transformation and load into a target table with a different table structure.
SQLite database:
CREATE TABLE sys_batch (
batch_id INTEGER PRIMARY KEY AUTOINCREMENT,
batch_name VARCHAR2 ( 100 ) );
CREATE TABLE sys_batch_tgt (
batch_id INTEGER PRIMARY KEY,
name_of_batch VARCHAR2 ( 100 ) );
Python code:
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.sql import select
from sqlalchemy.orm import Session
import bonobo
#import bonobo_sqlalchemy
engine = sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/coolstuff.db')
Base = automap_base()
Base.prepare(engine, reflect=True)
# Tables
sys_batch = Base.classes.sys_batch
sys_batch_tgt = Base.classes.sys_batch_tgt
#def get_services():
# return {'sqlalchemy.pgengine': sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/carauction.db')}
def extract_sys_batch():
conn = engine.connect()
s = select([sys_batch])
result = conn.execute(s)
#conn.close()
row = result.fetchone()
yield row
# convert incoming SQLAlchemy "ProxyRow" to a dict, so we can manipulate it.
def tfm_0(row):
tgt_row = {}
tgt_row['batch_id'] = row['batch_id']
tgt_row['name_of_batch'] = row['batch_name']
return tgt_row
def tfm_1(row):
row['batch_id'] = row['batch_id'] + 100
return row
def ldr_1(row):
# write to target
session = Session(engine)
thing = sys_batch_tgt(**row)
session.add(thing)
session.commit()
def get_graph(**options):
return bonobo.Graph(
extract_sys_batch,
tfm_0,
tfm_1,
ldr_1,
bonobo.PrettyPrinter(),
)
# The __main__ block actually execute the graph.
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
#each transformation is its own thread, so we have to open/close in there.
#conn = engine.connect()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**options)) #, services=get_services(**options)
#conn.close()
python python-3.x sqlalchemy
New contributor
Today I tried Bonobo + SQLAlchemy (for the first time), so it's bound to be full of bad practices. However, this is working code. I'm hoping for some feedback to for doing it correctly.
I started out using the SQLAlchemy extension for Bonobo, but I ran into limitations, so I wrote my own loader. My DB is SQLite3, so it doesn't support multiple connections.
What this does is read from one table, do a simple transformation and load into a target table with a different table structure.
SQLite database:
CREATE TABLE sys_batch (
batch_id INTEGER PRIMARY KEY AUTOINCREMENT,
batch_name VARCHAR2 ( 100 ) );
CREATE TABLE sys_batch_tgt (
batch_id INTEGER PRIMARY KEY,
name_of_batch VARCHAR2 ( 100 ) );
Python code:
import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.sql import select
from sqlalchemy.orm import Session
import bonobo
#import bonobo_sqlalchemy
engine = sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/coolstuff.db')
Base = automap_base()
Base.prepare(engine, reflect=True)
# Tables
sys_batch = Base.classes.sys_batch
sys_batch_tgt = Base.classes.sys_batch_tgt
#def get_services():
# return {'sqlalchemy.pgengine': sqlalchemy.create_engine('sqlite:///C:/Py/bonobo/carauction.db')}
def extract_sys_batch():
conn = engine.connect()
s = select([sys_batch])
result = conn.execute(s)
#conn.close()
row = result.fetchone()
yield row
# convert incoming SQLAlchemy "ProxyRow" to a dict, so we can manipulate it.
def tfm_0(row):
tgt_row = {}
tgt_row['batch_id'] = row['batch_id']
tgt_row['name_of_batch'] = row['batch_name']
return tgt_row
def tfm_1(row):
row['batch_id'] = row['batch_id'] + 100
return row
def ldr_1(row):
# write to target
session = Session(engine)
thing = sys_batch_tgt(**row)
session.add(thing)
session.commit()
def get_graph(**options):
return bonobo.Graph(
extract_sys_batch,
tfm_0,
tfm_1,
ldr_1,
bonobo.PrettyPrinter(),
)
# The __main__ block actually execute the graph.
if __name__ == '__main__':
parser = bonobo.get_argument_parser()
#each transformation is its own thread, so we have to open/close in there.
#conn = engine.connect()
with bonobo.parse_args(parser) as options:
bonobo.run(get_graph(**options)) #, services=get_services(**options)
#conn.close()
python python-3.x sqlalchemy
python python-3.x sqlalchemy
New contributor
New contributor
edited 5 mins ago
Jamal♦
30.2k11115226
30.2k11115226
New contributor
asked 9 hours ago
svenema
142
142
New contributor
New contributor
add a comment |
add a comment |
active
oldest
votes
Your Answer
StackExchange.ifUsing("editor", function () {
return StackExchange.using("mathjaxEditing", function () {
StackExchange.MarkdownEditor.creationCallbacks.add(function (editor, postfix) {
StackExchange.mathjaxEditing.prepareWmdForMathJax(editor, postfix, [["\$", "\$"]]);
});
});
}, "mathjax-editing");
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "196"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: false,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: null,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
svenema is a new contributor. Be nice, and check out our Code of Conduct.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fcodereview.stackexchange.com%2fquestions%2f209920%2floading-into-a-target-table%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
active
oldest
votes
active
oldest
votes
active
oldest
votes
active
oldest
votes
svenema is a new contributor. Be nice, and check out our Code of Conduct.
svenema is a new contributor. Be nice, and check out our Code of Conduct.
svenema is a new contributor. Be nice, and check out our Code of Conduct.
svenema is a new contributor. Be nice, and check out our Code of Conduct.
Thanks for contributing an answer to Code Review Stack Exchange!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
Use MathJax to format equations. MathJax reference.
To learn more, see our tips on writing great answers.
Some of your past answers have not been well-received, and you're in danger of being blocked from answering.
Please pay close attention to the following guidance:
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fcodereview.stackexchange.com%2fquestions%2f209920%2floading-into-a-target-table%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown