From 069c7eeea629a69d59470e12f7d86e3bcb225295 Mon Sep 17 00:00:00 2001 From: dmtolpeko Date: Fri, 1 Dec 2017 21:19:49 +0300 Subject: [PATCH] Sybase ASE to Oracle migration improved --- sqldata/sqlapibase.cpp | 36 +++++++++++ sqldata/sqlapibase.h | 7 ++- sqldata/sqlctapi.cpp | 96 +++++++++++++++++++--------- sqldata/sqlctapi.h | 4 ++ sqldata/sqldata.cpp | 4 +- sqldata/sqldata.h | 2 +- sqldata/sqldatacmd.cpp | 24 ++++--- sqldata/sqldb.cpp | 53 +++++++++++++--- sqldata/sqldb2api.cpp | 8 +++ sqldata/sqlociapi.cpp | 140 ++++++++++++++++++++++++++++------------- sqldata/sqlociapi.h | 6 +- sqldata/sqlsncapi.cpp | 88 +++++++++++--------------- sqldata/sqlsncapi.h | 14 +++-- 13 files changed, 328 insertions(+), 154 deletions(-) diff --git a/sqldata/sqlapibase.cpp b/sqldata/sqlapibase.cpp index 986cd5e..cec5d04 100644 --- a/sqldata/sqlapibase.cpp +++ b/sqldata/sqlapibase.cpp @@ -271,6 +271,42 @@ void SqlApiBase::SplitConnectionString(const char *conn, std::string &user, std: } } +void SqlApiBase::SplitConnectionString(const char *conn, std::string &user, std::string &pwd, std::string &server, std::string &db, std::string &port) +{ + if(conn == NULL) + return; + + std::string db_full; + + SplitConnectionString(conn, user, pwd, db_full); + + const char *start = db_full.c_str(); + + // Find : and , that denote the server port and the database name + const char *semi = strchr(start, ':'); + const char *comma = strchr(start, ','); + + const char *end = (semi != NULL) ? semi :comma; + + // Define server name + if(end != NULL) + server.assign(start, (size_t)(end - start)); + else + server = start; + + // Define port + if(semi != NULL) + { + if(comma != NULL && comma > semi) + port.assign(semi + 1, (size_t)(comma - semi - 1)); + else + port = semi + 1; + } + + if(comma != NULL) + db = Str::SkipSpaces(comma + 1); +} + // Build a condition to select objects from the catalog and exclude from selection void SqlApiBase::GetSelectionCriteria(const char *select, const char *exclude, const char *schema, const char *object, std::string &output, const char *default_schema, bool upper_case) diff --git a/sqldata/sqlapibase.h b/sqldata/sqlapibase.h index 6e98c05..e1e8613 100644 --- a/sqldata/sqlapibase.h +++ b/sqldata/sqlapibase.h @@ -72,7 +72,9 @@ struct SqlCol // Column contains binary data (MySQL) bool _binary; - + // Data fetched in UTF-16 + bool _nchar; + // Column data type name std::string _datatype_name; std::string _t_datatype_name; @@ -106,7 +108,7 @@ struct SqlCol SqlCol() { *_name = '\x0'; *_t_name = '\x0'; - _len = 0; _fetch_len = 0; _precision = 0; _scale = 0; _binary = false; + _len = 0; _fetch_len = 0; _precision = 0; _scale = 0; _binary = false; _nchar = false; _native_dt = 0; _native_fetch_dt = 0; _data = NULL; _ind2 = NULL; /*ind4 = NULL;*/ ind = NULL; _len_ind2 = NULL; _len_ind4 = NULL; _nullable = true; _lob = false; _lob_fetch_status = 0; @@ -642,6 +644,7 @@ class SqlApiBase // Split connection string to user, password and database void SplitConnectionString(const char *conn, std::string &user, std::string &pwd, std::string &db); + void SplitConnectionString(const char *conn, std::string &user, std::string &pwd, std::string &server, std::string &db, std::string &port); // Build a condition to select objects from the catalog static void GetSelectionCriteria(const char *select, const char *exclude, const char *schema, const char *object, std::string &output, const char *default_schema, bool upper_case); diff --git a/sqldata/sqlctapi.cpp b/sqldata/sqlctapi.cpp index 3c4ab29..a952d1d 100644 --- a/sqldata/sqlctapi.cpp +++ b/sqldata/sqlctapi.cpp @@ -141,6 +141,16 @@ int SqlCtApi::Init() } } +#else + _ct_dll = Os::LoadLibrary(CTLIB_DLL); + _cs_dll = Os::LoadLibrary(CSLIB_DLL); + + if(_ct_dll == NULL || _cs_dll == NULL) + { + char *error = Os::LoadLibraryError(); + if(error != NULL) + strcpy(_native_error_text, error); + } #endif // Get functions @@ -211,26 +221,7 @@ int SqlCtApi::Init() // Set the connection string in the API object void SqlCtApi::SetConnectionString(const char *conn) { - if(conn == NULL) - return; - - std::string db; - - SplitConnectionString(conn, _user, _pwd, db); - - const char *start = db.c_str(); - - // Find , that denotes the database name - const char *comma = strchr(start, ','); - - // Define server and database name - if(comma != NULL) - { - _server.assign(start, (size_t)(comma - start)); - _db = comma + 1; - } - else - _server = start; + SplitConnectionString(conn, _user, _pwd, _server, _db, _port); } // Connect to the database @@ -253,8 +244,26 @@ int SqlCtApi::Connect(size_t *time_spent) rc = _ct_con_props(_connection, CS_SET, CS_USERNAME, (CS_VOID*)_user.c_str(), CS_NULLTERM, NULL); rc = _ct_con_props(_connection, CS_SET, CS_PASSWORD, (CS_VOID*)_pwd.c_str(), CS_NULLTERM, NULL); + // Check if password encryption required + if(_parameters->GetTrue("-sybase_encrypted_password") != NULL) + { + CS_BOOL sec_encryption = CS_TRUE; + + // Sybase uses extended password encryption as the first preference, if the server cannot support extended password encryption, it uses normal password encryption + rc = _ct_con_props(_connection, CS_SET, CS_SEC_EXTENDED_ENCRYPTION, (CS_VOID*)&sec_encryption, CS_UNUSED, NULL); + rc = _ct_con_props(_connection, CS_SET, CS_SEC_ENCRYPTION, (CS_VOID*)&sec_encryption, CS_UNUSED, NULL); + } + const char *server = _server.empty() ? NULL : _server.c_str(); + // Check if port is specified + if(!_port.empty()) + { + // Use format "server port" + std::string serveraddr = _server + " " + _port; + rc = _ct_con_props(_connection, CS_SET, CS_SERVERADDR, (CS_VOID*)serveraddr.c_str(), CS_NULLTERM, NULL); + } + // Connect to the server rc = _ct_connect(_connection, (CS_CHAR*)server, CS_NULLTERM); @@ -507,6 +516,11 @@ int SqlCtApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memor // Get column information for(int i = 0; i < _cursor_cols_count; i++) { + // Note that CHAR and VARCHAR both have CS_CHAR_TYPE, and you cannot distinguish them using CS_FMT_PADBLANK and CS_FMT_PADNULL in fmt.format + // as it is always 0 (CS_FMT_UNUSED) in ct_describe (Not applicable) + + // CS_LONGCHAR_TYPE is returned for VARCHAR > 255 (max length is 32K since ASE 12.5) + rc = _ct_describe(_cursor_cmd, i + 1, &fmt[i]); // Copy the column name @@ -521,9 +535,9 @@ int SqlCtApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memor // Get column length for character and binary strings _cursor_cols[i]._len = (size_t)fmt[i].maxlength; - - // TEXT, Sybase ASE 16 returns size 32768, change to 1M - if(_cursor_cols[i]._native_dt == CS_TEXT_TYPE) + + // For TEXT and UNITEXT Sybase ASE 16 returns size 32768, change to 1M + if(_cursor_cols[i]._native_dt == CS_TEXT_TYPE || _cursor_cols[i]._native_dt == CS_UNITEXT_TYPE) _cursor_cols[i]._len = 1048576; row_size += _cursor_cols[i]._len; @@ -551,8 +565,8 @@ int SqlCtApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memor // Allocate buffers for each column for(int i = 0; i < _cursor_cols_count; i++) { - // CHAR data type - if(_cursor_cols[i]._native_dt == CS_CHAR_TYPE) + // CHAR and VARCHAR data types; CS_CHAR_TYPE for VARCHAR <= 255, and CS_LONGCHAR_TYPE for VARCHAR < 32K + if(_cursor_cols[i]._native_dt == CS_CHAR_TYPE || _cursor_cols[i]._native_dt == CS_LONGCHAR_TYPE) { // Do not bind to null-terminating string as zero byte will be included to length indicator _cursor_cols[i]._native_fetch_dt = _cursor_cols[i]._native_dt; @@ -561,6 +575,15 @@ int SqlCtApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memor _cursor_cols[i]._data = new char[_cursor_cols[i]._fetch_len * _cursor_allocated_rows]; } else + // BINARY data type + if(_cursor_cols[i]._native_dt == CS_BINARY_TYPE) + { + _cursor_cols[i]._native_fetch_dt = _cursor_cols[i]._native_dt; + _cursor_cols[i]._fetch_len = _cursor_cols[i]._len; + + _cursor_cols[i]._data = new char[_cursor_cols[i]._fetch_len * _cursor_allocated_rows]; + } + else // BIGINT data type if(_cursor_cols[i]._native_dt == CS_BIGINT_TYPE) { @@ -759,6 +782,20 @@ int SqlCtApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memor fmt[i].datatype = CS_CHAR_TYPE; fmt[i].maxlength = (CS_INT)_cursor_cols[i]._fetch_len; + } + else + // UNITEXT + if(_cursor_cols[i]._native_dt == CS_UNITEXT_TYPE) + { + // Data fetched as UTF-16 i.e. 0x00 byte goes first for first 127 ASCII characters + _cursor_cols[i]._native_fetch_dt = CS_UNICHAR_TYPE; + _cursor_cols[i]._nchar = true; + + _cursor_cols[i]._fetch_len = _cursor_cols[i]._len; + _cursor_cols[i]._data = new char[_cursor_cols[i]._fetch_len * _cursor_allocated_rows]; + + fmt[i].datatype = CS_UNICHAR_TYPE; + fmt[i].maxlength = (CS_INT)_cursor_cols[i]._fetch_len; } // Bind the data to array @@ -1192,7 +1229,8 @@ int SqlCtApi::ReadTableColumns(std::string &condition) _table_columns.push_back(col_meta); } - rc = Fetch(&rows_fetched, &time_read); + if(rc != 100) + rc = Fetch(&rows_fetched, &time_read); // No more rows if(rc == 100) @@ -1345,7 +1383,8 @@ int SqlCtApi::ReadIndexes(std::string &condition) } } - rc = Fetch(&rows_fetched, &time_read); + if(rc != 100) + rc = Fetch(&rows_fetched, &time_read); // No more rows if(rc == 100) @@ -1530,7 +1569,8 @@ int SqlCtApi::ReadReferences(std::string &condition) } } - rc = Fetch(&rows_fetched, &time_read); + if(rc != 100) + rc = Fetch(&rows_fetched, &time_read); // No more rows if(rc == 100) diff --git a/sqldata/sqlctapi.h b/sqldata/sqlctapi.h index bae4f35..429242a 100644 --- a/sqldata/sqlctapi.h +++ b/sqldata/sqlctapi.h @@ -35,6 +35,9 @@ #elif defined(WIN32) #define CTLIB_DLL "libsybct.dll" #define CSLIB_DLL "libsybcs.dll" +#else +#define CTLIB_DLL "libsybct64.so" +#define CSLIB_DLL "libsybcs64.so" #endif // Sybase ASE 12.5 DLLs @@ -79,6 +82,7 @@ class SqlCtApi : public SqlApiBase std::string _user; std::string _pwd; std::string _server; + std::string _port; std::string _db; // Open cursor command diff --git a/sqldata/sqldata.cpp b/sqldata/sqldata.cpp index ecc4a51..4eeb725 100644 --- a/sqldata/sqldata.cpp +++ b/sqldata/sqldata.cpp @@ -2359,7 +2359,9 @@ void SqlData::GetIdentityMetaTask(SqlColMeta &col) char start_str[11]; char inc_str[11]; - sprintf(start_str, "%d", col.id_next); + int start = (col.id_next > 0) ? col.id_next : col.id_start; + + sprintf(start_str, "%d", start); sprintf(inc_str, "%d", col.id_inc); // Create sequence diff --git a/sqldata/sqldata.h b/sqldata/sqldata.h index 5042f95..be6a186 100644 --- a/sqldata/sqldata.h +++ b/sqldata/sqldata.h @@ -34,7 +34,7 @@ #include "applog.h" #define SQLDATA_NAME "SQLines Data" -#define SQLDATA_VERSION_NUMBER "3.1.317" +#define SQLDATA_VERSION_NUMBER "3.1.703" #if defined(_WIN64) #define SQLDATA_VERSION SQLDATA_NAME ## " " ## SQLDATA_VERSION_NUMBER ##" x64" diff --git a/sqldata/sqldatacmd.cpp b/sqldata/sqldatacmd.cpp index 7741370..b60436e 100644 --- a/sqldata/sqldatacmd.cpp +++ b/sqldata/sqldatacmd.cpp @@ -930,12 +930,11 @@ void SqlDataCmd::CallbackValidationRows(SqlDataReply *reply) // Show differences if(reply->_int1 != 0) - { _log.Log("\n Not Equal: %d row%s, %d of %d column%s (%s)", reply->_int1, SUFFIX(reply->_int1), reply->_int2, reply->_int3, SUFFIX(reply->_int3), reply->data2); - _log.Log("\n Source query: %s", reply->s_sql_l.c_str()); - _log.Log("\n Target query: %s", reply->t_sql_l.c_str()); - } + + _log.Log("\n Source query: %s", reply->s_sql_l.c_str()); + _log.Log("\n Target query: %s", reply->t_sql_l.c_str()); } else // All tables validated @@ -1145,8 +1144,19 @@ int SqlDataCmd::ReadMetadata() // Create queues to transfer schema metadata std::string meta_filter = _t; + if(_t.empty()) - meta_filter = "*"; + { + int c = 0; + for(std::list::iterator i = avail_tables.begin(); i != avail_tables.end(); i++) + { + if(c > 0) + meta_filter += ","; + + meta_filter += (*i); + c++; + } + } _sqlData.CreateMetadataQueues(meta_filter, _texcl); @@ -1790,13 +1800,13 @@ void SqlDataCmd::PrintHowToUse() printf("\n\nExample:"); printf("\n\nTransfer table cities from Oracle to SQL Server"); #if defined(WIN32) || defined(_WIN64) - printf("\n\n sqldata.exe -sd=oracle,scott/tiger@orcl -td=sql,trusted@srv1.hr -t=cities"); + printf("\n\n sqldata.exe -sd=oracle,scott/tiger@orcl -td=sql,trusted@srv1,hr -t=cities"); printf("\n"); printf("\nRun sqldata_w.exe to launch a GUI version of SQLData.\n"); printf("\nPress any key to continue...\n"); _getch(); #else - printf("\n\n ./sqldata -sd=oracle,scott/tiger@orcl -td=sql,trusted@srv1.hr -t=cities"); + printf("\n\n ./sqldata -sd=oracle,scott/tiger@orcl -td=sql,trusted@srv1,hr -t=cities"); printf("\n\n"); #endif } diff --git a/sqldata/sqldb.cpp b/sqldata/sqldb.cpp index 54946c5..895db9c 100644 --- a/sqldata/sqldb.cpp +++ b/sqldata/sqldb.cpp @@ -1291,6 +1291,8 @@ int SqlDb::GenerateCreateTable(SqlCol *s_cols, const char *s_table, const char * else // Oracle VARCHAR2 (SQLT_CHR) ODBC SQL_VARCHAR if((source_type == SQLDATA_ORACLE && s_cols[i]._native_dt == SQLT_CHR) || + // Sybase ASE CHAR and VARCHAR (CS_CHAR_TYPE for VARCHAR <= 255, and CS_LONGCHAR_TYPE for VARCHAR < 32K) + (source_type == SQLDATA_SYBASE && (s_cols[i]._native_dt == CS_CHAR_TYPE || s_cols[i]._native_dt == CS_LONGCHAR_TYPE)) || // MySQL VARCHAR (source_type == SQLDATA_MYSQL && s_cols[i]._native_dt == MYSQL_TYPE_VAR_STRING) || // SQL Server, DB2, Informix, Sybase ASA VARCHAR @@ -1355,8 +1357,6 @@ int SqlDb::GenerateCreateTable(SqlCol *s_cols, const char *s_table, const char * if((source_type == SQLDATA_ORACLE && s_cols[i]._native_dt == SQLT_AFC) || // SQL Server CHAR (source_type == SQLDATA_SQL_SERVER && s_cols[i]._native_dt == SQL_CHAR) || - // Sybase ASE CHAR - (source_type == SQLDATA_SYBASE && s_cols[i]._native_dt == CS_CHAR_TYPE) || // Informix CHAR (source_type == SQLDATA_INFORMIX && s_cols[i]._native_dt == SQL_CHAR) || // DB2 CHAR @@ -1864,8 +1864,17 @@ int SqlDb::GenerateCreateTable(SqlCol *s_cols, const char *s_table, const char * } else // SQL Server BINARY - if(source_type == SQLDATA_SQL_SERVER && s_cols[i]._native_dt == SQL_BINARY) + if((source_type == SQLDATA_SQL_SERVER && s_cols[i]._native_dt == SQL_BINARY) || + // Sybase ASE BINARY + (source_type == SQLDATA_SYBASE && s_cols[i]._native_dt == CS_BINARY_TYPE)) { + if(target_type == SQLDATA_ORACLE) + { + sql += "RAW("; + sql += Str::IntToString((int)s_cols[i]._len, int1); + sql += ")"; + } + else if(target_type == SQLDATA_MYSQL) { sql += "BINARY("; @@ -1931,8 +1940,8 @@ int SqlDb::GenerateCreateTable(SqlCol *s_cols, const char *s_table, const char * // Informix TEXT, Sybase ASA LONG VARCHAR ((source_type == SQLDATA_INFORMIX || source_type == SQLDATA_ASA || source_type == SQLDATA_ODBC) && s_cols[i]._native_dt == SQL_LONGVARCHAR) || - // DB2 CLOB with code -99 - (source_type == SQLDATA_DB2 && s_cols[i]._native_dt == -99) || + // DB2 CLOB with code -99, LONG VARCHAR with code -1 + (source_type == SQLDATA_DB2 && (s_cols[i]._native_dt == -99 || s_cols[i]._native_dt == -1)) || // Informix CLOB with code -103 (source_type == SQLDATA_INFORMIX && s_cols[i]._native_dt == -103) || // MySQL CLOB @@ -1952,9 +1961,11 @@ int SqlDb::GenerateCreateTable(SqlCol *s_cols, const char *s_table, const char * } else // SQL Server NTEXT, Sybase ASA LONG NVARCHAR - if((source_type == SQLDATA_SQL_SERVER || source_type == SQLDATA_INFORMIX || + if(((source_type == SQLDATA_SQL_SERVER || source_type == SQLDATA_INFORMIX || source_type == SQLDATA_ASA || source_type == SQLDATA_ODBC) - && s_cols[i]._native_dt == SQL_WLONGVARCHAR) + && s_cols[i]._native_dt == SQL_WLONGVARCHAR) || + // Sybase ASE UNITEXT + (source_type == SQLDATA_SYBASE && s_cols[i]._native_dt == CS_UNITEXT_TYPE)) { if(target_type == SQLDATA_SQL_SERVER) sql += "NVARCHAR(MAX)"; @@ -2625,11 +2636,11 @@ int SqlDb::ValidateRows(SqlDataReply &reply) { strncpy(reply.data2, diff_cols_list.c_str(), 1023); reply.data2[1023] = '\x0'; - - reply.s_sql_l = s_select; - reply.t_sql_l = t_select; } + reply.s_sql_l = s_select; + reply.t_sql_l = t_select; + reply._s_bigint1 = s_all_bytes; reply._t_bigint1 = t_all_bytes; @@ -3351,6 +3362,19 @@ int SqlDb::BuildQuery(std::string &s_query, std::string &t_query, const char *s_ } } else + // Oracle XMLTYPE column that fetched as object type in OCI + if(source_type == SQLDATA_ORACLE && d != NULL && !_stricmp(d, "XMLTYPE")) + { + s_query += "XMLSERIALIZE(CONTENT \""; + s_query += c; + s_query += "\" AS CLOB) AS "; + s_query += c; + + t_query += c; + + column_added = true; + } + else // Oracle stores quoted columns without " but in the same case as they specified in CREATE TABLE if(source_type == SQLDATA_ORACLE) { @@ -3365,6 +3389,15 @@ int SqlDb::BuildQuery(std::string &s_query, std::string &t_query, const char *s_ t_query += c; t_query += "]"; } + else + if(target_type == SQLDATA_MYSQL) + { + t_query += '`'; + t_query += c; + t_query += "`"; + } + else + t_query += c; column_added = true; } diff --git a/sqldata/sqldb2api.cpp b/sqldata/sqldb2api.cpp index 8b0defe..abf031c 100644 --- a/sqldata/sqldb2api.cpp +++ b/sqldata/sqldb2api.cpp @@ -426,6 +426,14 @@ int SqlDb2Api::OpenCursor(const char *query, size_t buffer_rows, int buffer_memo _cursor_cols[i]._data = new char[_cursor_cols[i]._fetch_len * _cursor_allocated_rows]; } else + // LONG VARCHAR, len is 32700 + if(_cursor_cols[i]._native_dt == -1) + { + _cursor_cols[i]._native_fetch_dt = SQL_C_CHAR; + _cursor_cols[i]._fetch_len = _cursor_cols[i]._len + 1; + _cursor_cols[i]._data = new char[_cursor_cols[i]._fetch_len * _cursor_allocated_rows]; + } + else // Data type GRAPHIC with code -95 (fixed-length UTF-16) or VARGRAPHIC with code -96 (variable-length UTF-16) if(_cursor_cols[i]._native_dt == -95 || _cursor_cols[i]._native_dt == -96) { diff --git a/sqldata/sqlociapi.cpp b/sqldata/sqlociapi.cpp index 2228b28..46aa5bf 100644 --- a/sqldata/sqlociapi.cpp +++ b/sqldata/sqlociapi.cpp @@ -42,10 +42,6 @@ SqlOciApi::SqlOciApi() _stmtp_cursor = NULL; _stmtp_insert = NULL; - _user[0] = '\x0'; - _pwd[0] = '\x0'; - _db[0] = '\x0'; - _charset_id = 0; _ociArrayDescriptorAlloc = NULL; @@ -87,9 +83,16 @@ int SqlOciApi::Init() const char *oci_lib_param = (_parameters != NULL) ? _parameters->Get("-oci_lib") : NULL; const char *oci_lib = (oci_lib_param != NULL)? oci_lib_param : OCI_DLL; + const char *nls_lang = NULL; + + if(_parameters != NULL) + nls_lang = _parameters->Get("-oracle_nls_lang"); #if defined(WIN32) || defined(WIN64) + if(nls_lang != NULL) + _putenv((std::string("NLS_LANG=") + nls_lang).c_str()); + else // Force UTF-8 codepage at the client side if the target loader supports it if(_target_api_provider != NULL && _target_api_provider->IsTargetUtf8LoadSupported()) _putenv("NLS_LANG=American_America.AL32UTF8"); @@ -210,36 +213,7 @@ int SqlOciApi::Init() // Set the connection string in the API object void SqlOciApi::SetConnectionString(const char *conn) { - if(conn == NULL) - return; - - // Find @ that separates user/password from tnsname or host:port/sid - const char *amp = strchr(conn, '@'); - const char *sl = strchr(conn, '/'); - - // Set server info - if(amp != NULL) - strcpy(_db, amp + 1); - - if(amp == NULL) - amp = conn + strlen(conn); - - // Define the end of the user name - const char *end = (sl == NULL || amp < sl) ? amp : sl; - - strncpy(_user, conn, (size_t)(end - conn)); - _user[end - conn] = '\x0'; - - // Define password - if(sl != NULL && amp > sl) - { - strncpy(_pwd, sl + 1, (size_t)(amp - sl - 1)); - _pwd[amp - sl - 1] = '\x0'; - } - else - *_pwd = '\x0'; - - return; + SplitConnectionString(conn, _user, _pwd, _db); } // Connect to the database @@ -271,7 +245,7 @@ int SqlOciApi::Connect(size_t *time_spent) if(rc < 0) return rc; - rc = _ociServerAttach(_srvhp, _errhp, (text*)_db, (sb4)strlen(_db), 0); + rc = _ociServerAttach(_srvhp, _errhp, (text*)_db.c_str(), (sb4)_db.length(), 0); if(rc < 0) { @@ -294,8 +268,8 @@ int SqlOciApi::Connect(size_t *time_spent) return rc; // Set login information - rc = _ociAttrSet(_authp, OCI_HTYPE_SESSION, _user, (ub4)strlen(_user), OCI_ATTR_USERNAME, _errhp); - rc = _ociAttrSet(_authp, OCI_HTYPE_SESSION, _pwd, (ub4)strlen(_pwd), OCI_ATTR_PASSWORD, _errhp); + rc = _ociAttrSet(_authp, OCI_HTYPE_SESSION, (void*)_user.c_str(), (ub4)_user.length(), OCI_ATTR_USERNAME, _errhp); + rc = _ociAttrSet(_authp, OCI_HTYPE_SESSION, (void*)_pwd.c_str(), (ub4)_pwd.length(), OCI_ATTR_PASSWORD, _errhp); rc = _ociSessionBegin(_svchp, _errhp, _authp, OCI_CRED_RDBMS, OCI_DEFAULT); @@ -557,7 +531,7 @@ int SqlOciApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memo _cursor_lob_exists = true; } } - + else // For LONG data type, 0 size is returned if(_cursor_cols[i]._native_dt == SQLT_LNG) { @@ -569,6 +543,18 @@ int SqlOciApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memo else _cursor_cols[i]._len = 4000; } + else + // Object type including XMLTYPE + if(_cursor_cols[i]._native_dt == SQLT_NTY) + { + if(_cursor_fetch_lob_as_varchar) + { + _bind_long_inplace = 32000; + _cursor_cols[i]._len = _bind_long_inplace; + } + else + _cursor_cols[i]._len = 4000; + } row_size += _cursor_cols[i]._len; @@ -717,6 +703,14 @@ int SqlOciApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memo } } else + // Object type + if(_cursor_cols[i]._native_dt == SQLT_NTY) + { + _cursor_cols[i]._native_fetch_dt = SQLT_STR; + _cursor_cols[i]._fetch_len = _cursor_cols[i]._len; + _cursor_cols[i]._data = new char[_cursor_cols[i]._fetch_len * _cursor_allocated_rows]; + } + else // ROWID data type if(_cursor_cols[i]._native_dt == SQLT_RDD) { @@ -953,7 +947,7 @@ int SqlOciApi::GetAvailableTables(std::string &select, std::string &exclude, std::string condition; // Get a condition to select objects from the catalog - GetSelectionCriteria(select.c_str(), exclude.c_str(), "owner", "table_name", condition, _user, true); + GetSelectionCriteria(select.c_str(), exclude.c_str(), "owner", "table_name", condition, _user.c_str(), true); // Build the query std::string query = "SELECT owner, table_name FROM all_tables WHERE"; @@ -1052,8 +1046,8 @@ int SqlOciApi::ReadSchema(const char *select, const char *exclude, bool read_cns ClearSchema(); // Build WHERE clause to select rows from catalog - GetSelectionCriteria(select, exclude, "owner", "table_name", selection, _user, true); - GetSelectionCriteria(select, exclude, "table_owner", "table_name", selection2, _user, true); + GetSelectionCriteria(select, exclude, "owner", "table_name", selection, _user.c_str(), true); + GetSelectionCriteria(select, exclude, "table_owner", "table_name", selection2, _user.c_str(), true); // Exclude system and Oracle specific schemas if *.* condition is set if(select != NULL && strcmp(select, "*.*") == 0 && selection.empty()) @@ -2546,6 +2540,7 @@ int SqlOciApi::InitBulkTransfer(const char *table, size_t col_count, size_t allo _ins_cols[i]._fetch_len = s_cols[i]._fetch_len; _ins_cols[i]._lob = s_cols[i]._lob; + _ins_cols[i]._nchar = s_cols[i]._nchar; // Forward all Oracle data types if(_source_api_type == SQLDATA_ORACLE) @@ -2557,7 +2552,7 @@ int SqlOciApi::InitBulkTransfer(const char *table, size_t col_count, size_t allo // SQL Server, Informix, DB2 and Sybase ASA types fetched as CHAR ((_source_api_type == SQLDATA_SQL_SERVER || _source_api_type == SQLDATA_INFORMIX || _source_api_type == SQLDATA_DB2 || _source_api_type == SQLDATA_ASA) - && s_cols[i]._native_fetch_dt == SQL_C_CHAR) || + && (s_cols[i]._native_fetch_dt == SQL_C_CHAR || s_cols[i]._native_fetch_dt == SQL_C_WCHAR)) || // MySQL data types bound to string except TEXT and BLOB (_source_api_type == SQLDATA_MYSQL && s_cols[i]._lob == false)) { @@ -2571,13 +2566,22 @@ int SqlOciApi::InitBulkTransfer(const char *table, size_t col_count, size_t allo // SQL Server, Informix, DB2 and Sybase ASA types fetched as BINARY ((_source_api_type == SQLDATA_SQL_SERVER || _source_api_type == SQLDATA_INFORMIX || _source_api_type == SQLDATA_DB2 || _source_api_type == SQLDATA_ASA) - && s_cols[i]._native_fetch_dt == SQL_C_BINARY)) + && s_cols[i]._native_fetch_dt == SQL_C_BINARY) || + // Sybase ASE BINARY + (_source_api_type == SQLDATA_SYBASE && s_cols[i]._native_fetch_dt == CS_BINARY_TYPE)) { _ins_cols[i]._native_dt = SQLT_BIN; } else - // Sybase CHAR - if(_source_api_type == SQLDATA_SYBASE && s_cols[i]._native_fetch_dt == CS_CHAR_TYPE) + // Sybase CHAR and VARCHAR + if(_source_api_type == SQLDATA_SYBASE && (s_cols[i]._native_fetch_dt == CS_CHAR_TYPE || s_cols[i]._native_fetch_dt == CS_LONGCHAR_TYPE)) + { + // Bind without terminating NULL (it required in case of SQLT_STR) + _ins_cols[i]._native_dt = SQLT_AFC; + } + else + // Sybase UNICHAR + if(_source_api_type == SQLDATA_SYBASE && s_cols[i]._native_fetch_dt == CS_UNICHAR_TYPE) { // Bind without terminating NULL (it required in case of SQLT_STR) _ins_cols[i]._native_dt = SQLT_AFC; @@ -2642,6 +2646,23 @@ int SqlOciApi::InitBulkTransfer(const char *table, size_t col_count, size_t allo data = &_ins_cols[i]._data; } + else + // Sybase INITEXT UTF-16 + if(_source_api_type == SQLDATA_SYBASE && s_cols[i]._native_fetch_dt == CS_UNITEXT_TYPE) + { + _ins_cols[i]._native_dt = SQLT_CLOB; + _ins_cols[i]._fetch_len = 0; + _ins_cols[i]._lob = true; + + // Use data field to store pointer to the LOB locator + rc = _ociDescriptorAlloc(_envhp, (void**)&_ins_cols[i]._data, OCI_DTYPE_LOB, 0, NULL); + + // Temporary LOB for inserting data + rc = _ociLobCreateTemporary(_svchp, _errhp, (OCILobLocator*)_ins_cols[i]._data, 0, SQLCS_NCHAR, + OCI_TEMP_CLOB, OCI_ATTR_NOCACHE, OCI_DURATION_SESSION); + + data = &_ins_cols[i]._data; + } OCIBind *bindpp = NULL; @@ -2649,6 +2670,16 @@ int SqlOciApi::InitBulkTransfer(const char *table, size_t col_count, size_t allo rc = _ociBindByPos(_stmtp_insert, &bindpp, _errhp, (ub4)(i + 1), data, (sb4)_ins_cols[i]._fetch_len, (ub2)_ins_cols[i]._native_dt, ind, (ub2*)_ins_cols[i]._len_ind2, NULL, 0, NULL, OCI_DEFAULT); + // Data extracted in UTF-16 + if(_ins_cols[i]._nchar) + { + ub1 cform = SQLCS_NCHAR; + ub2 csid = OCI_UTF16ID; + + rc = _ociAttrSet(bindpp, OCI_HTYPE_BIND, (void*)&cform, 0, OCI_ATTR_CHARSET_FORM, _errhp); + rc = _ociAttrSet(bindpp, OCI_HTYPE_BIND, (void *)&csid, 0, OCI_ATTR_CHARSET_ID, _errhp); + } + if(rc == -1) { SetError(); @@ -2762,6 +2793,16 @@ int SqlOciApi::TransferRows(SqlCol *s_cols, int rows_fetched, int *rows_written, _ins_cols[k]._ind2[i] = 0; _ins_cols[k]._len_ind2[i] = (short)ins_len; } + // Sybase ASE DATE fetched as full 26 byte timestamp string and must be truncated to Oracle DATE length + // otherwise Oracle returns "ORA-01830: date format picture ends before converting entire input string" + else + if(_source_api_type == SQLDATA_SYBASE && s_cols[k]._native_dt == CS_DATE_TYPE) + { + short len = _ins_cols[k]._len_ind2[i]; + + if(len == 26) + _ins_cols[k]._len_ind2[i] = 19; + } else // LOB column if(_source_api_type != SQLDATA_ORACLE && s_cols[k]._lob == true && _ins_cols[k]._ind2[i] != -1) @@ -2828,6 +2869,15 @@ int SqlOciApi::TransferRows(SqlCol *s_cols, int rows_fetched, int *rows_written, bytes += lob_size; } + else + // Correct the size for UTF-16 data since buffer in bytes while Oracle OCI requires size in characters + if(_ins_cols[k]._nchar) + { + short len = _ins_cols[k]._len_ind2[i]; + + if(len > 2 && len % 2 == 0) + _ins_cols[k]._len_ind2[i] = len/2; + } } } diff --git a/sqldata/sqlociapi.h b/sqldata/sqlociapi.h index a896610..4fd7141 100644 --- a/sqldata/sqlociapi.h +++ b/sqldata/sqlociapi.h @@ -86,9 +86,9 @@ class SqlOciApi : public SqlApiBase OCIStmt *_stmtp_insert; // Connection information - char _user[256]; - char _pwd[256]; - char _db[256]; + std::string _user; + std::string _pwd; + std::string _db; // Character set ID (873 - AL32UTF8) ub2 _charset_id; diff --git a/sqldata/sqlsncapi.cpp b/sqldata/sqlsncapi.cpp index a2919b0..98e3374 100644 --- a/sqldata/sqlsncapi.cpp +++ b/sqldata/sqlsncapi.cpp @@ -32,10 +32,6 @@ SqlSncApi::SqlSncApi() _hdbc = SQL_NULL_HANDLE; _hstmt_cursor = SQL_NULL_HANDLE; - *_user = '\x0'; - *_pwd = '\x0'; - *_server = '\x0'; - *_db = '\x0'; _trusted = false; _cursor_fetched = 0; @@ -43,6 +39,7 @@ SqlSncApi::SqlSncApi() _bcp_cols_count = 0; _bcp_cols = NULL; _bcp_lob_exists = false; + _bcp_codepage = -2; // -1 is used by BCPFILECP_RAW _error = 0; _error_text[0] = '\x0'; @@ -94,6 +91,16 @@ int SqlSncApi::Init() _dll_odbc = LoadLibraryEx(SQLSRV_ODBC_DLL, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); _dll = LoadLibraryEx(dll.c_str(), NULL, LOAD_WITH_ALTERED_SEARCH_PATH); +#else + _dll_odbc = Os::LoadLibrary(SQLSERV_DLL); + _dll = _dll_odbc; + + if(_dll_odbc == NULL || _dll == NULL) + { + char *error = Os::LoadLibraryError(); + if(error != NULL) + strcpy(_native_error_text, error); + } #endif // Get functions @@ -142,59 +149,20 @@ int SqlSncApi::Init() return -1; } + // Initialize options + if(_parameters != NULL) + _bcp_codepage = _parameters->GetInt("-bcp_codepage", -2); // -1 is used by BCPFILECP_RAW + return 0; } // Set the connection string in the API object void SqlSncApi::SetConnectionString(const char *conn) { - if(conn == NULL) - return; - - // Find @ that separates user/password from server name - const char *amp = strchr(conn, '@'); - const char *sl = strchr(conn, '/'); - - // Set server info - if(amp != NULL) - { - // Find database name - const char *comma = strchr(amp + 1, ','); - - if(comma != NULL) - { - strncpy(_server, amp + 1, (size_t)(comma - amp - 1)); - _server[comma - amp - 1] = '\x0'; - - strcpy(_db, Str::SkipSpaces(comma + 1)); - } - else - { - strcpy(_server, amp + 1); - *_db = '\x0'; - } - } - - if(amp == NULL) - amp = conn + strlen(conn); - - // Define the end of the user name - const char *end = (sl == NULL || amp < sl) ? amp : sl; - - strncpy(_user, conn, (size_t)(end - conn)); - _user[end - conn] = '\x0'; - - // Define password - if(sl != NULL && amp > sl) - { - strncpy(_pwd, sl + 1, (size_t)(amp - sl - 1)); - _pwd[amp - sl - 1] = '\x0'; - } - else - *_pwd = '\x0'; + SplitConnectionString(conn, _user, _pwd, _server, _db, _port); // Check for trusted connection - _trusted = (_stricmp(_user, "trusted") == 0) ? true : false; + _trusted = (_stricmp(_user.c_str(), "trusted") == 0) ? true : false; return; } @@ -226,15 +194,22 @@ int SqlSncApi::Connect(size_t *time_spent) std::string conn = _driver; // Build connection string, add server - if(*_server) + if(!_server.empty()) { conn += "Server="; conn += _server; + + if(!_port.empty()) + { + conn += ","; + conn += _port; + } + conn += ";"; } // Add server - if(*_db) + if(!_db.empty()) { conn += "Database="; conn += _db; @@ -526,8 +501,9 @@ int SqlSncApi::OpenCursor(const char *query, size_t buffer_rows, int buffer_memo // Data type NCHAR or NVARCHAR if(_cursor_cols[i]._native_dt == SQL_WCHAR || _cursor_cols[i]._native_dt == SQL_WVARCHAR) { - // MySQL does not support UCS2 in LOAD DATA INFILE, let's convert to ASCII - if(_target_api_provider != NULL && _target_api_provider->GetType() == SQLDATA_MYSQL) + // MySQL does not support UCS2 in LOAD DATA INFILE, let's convert to ASCII; also convert to ASCII for target Oracle + if(_target_api_provider != NULL && + (_target_api_provider->GetType() == SQLDATA_MYSQL || _target_api_provider->GetType() == SQLDATA_ORACLE)) { _cursor_cols[i]._native_fetch_dt = SQL_C_CHAR; _cursor_cols[i]._fetch_len = _cursor_cols[i]._len + 1; @@ -929,6 +905,12 @@ int SqlSncApi::InitBulkTransfer(const char *table, size_t col_count, size_t allo // Use IDENTITY values from the source table rc = _bcp_control(_hdbc, BCPKEEPIDENTITY, (void*)TRUE); + // Set the BCP codepage defined by the user. + // By default Windows locale defines input data codepage that can cause problems when Windows and table locale does not match + // For some reason specifying BCPFILECP_RAW (-1) still causes conversion and cannot be used as default + if(_bcp_codepage != -2) + rc = _bcp_control(_hdbc, BCPFILECP, (void*)_bcp_codepage); + _bcp_cols_count = col_count; _bcp_cols = new SqlCol[col_count]; diff --git a/sqldata/sqlsncapi.h b/sqldata/sqlsncapi.h index dc2474d..f28ceb2 100644 --- a/sqldata/sqlsncapi.h +++ b/sqldata/sqlsncapi.h @@ -39,6 +39,8 @@ #define SQLSRV_ODBC_DLL "odbc32.dll" +#define SQLSERV_DLL "libsqlncli.so" + #define SQLSRV_DLL_LOAD_ERROR "Loading SQL Server library:" // ODBC API Functions @@ -74,10 +76,11 @@ class SqlSncApi : public SqlApiBase SQLHANDLE _hstmt_cursor; // Connection information - char _user[256]; - char _pwd[256]; - char _server[256]; - char _db[256]; + std::string _user; + std::string _pwd; + std::string _server; + std::string _port; + std::string _db; bool _trusted; std::string _driver; @@ -92,6 +95,9 @@ class SqlSncApi : public SqlApiBase // Exist LOB columns to be sent using bcp_moretext bool _bcp_lob_exists; + // Codepage to use for BCP + int _bcp_codepage; + // SQL Server Native Client DDL (for Bulk Copy API) #if defined(WIN32) || defined(_WIN64) HMODULE _dll;