access.log analysis

We're using awstat to analyse the access.log statistics.

Weblogic JMS pause and resume consumption

This program is an example of connecting to weblogic and manipulating the JMS queues.
The example here, pauses and resumes the consumption of a selected queue.


/*
* $Revision: $
* $Date: $
* $Author: $
*/

package com.test;

import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Hashtable;
import java.util.Set;
import java.util.Vector;

import javax.management.MBeanServerConnection;
import javax.management.MalformedObjectNameException;
import javax.management.ObjectName;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class QueueManipulator {
Logger logger = LoggerFactory.getLogger(this.getClass());
public static final String SYSTEM_OSB = "OSB";

public static void main(String[] args) {
QueueManipulator qm = new QueueManipulator();
qm.run();
}

public void run() {
String[] queues = { "PurchaseRequisitionCreateQueue" };
// Pause
updatePauseStatus(queues, true);
// resume
updatePauseStatus(queues, false);
}

private Collection updatePauseStatus(String[] queueNames,
boolean pause) {
Vector updatedQueues = new Vector();
Collection serverList;
// Connection details
String jmsModuleName, username, password;
serverList = new ArrayList();
serverList.add(new Server("ms_soa1", "osb1tst2:7001"));
// serverList.add(new Server("ms_soa2" ,"soa2tst2:7001"));
jmsModuleName = "ApplicationModule";
username = "khyland";
password = "Password1";

for (Server server : serverList) {
JMXConnector jmxCon = null;
try {
jmxCon = getJMXConnector(server, username, password);
jmxCon.connect();
MBeanServerConnection con = jmxCon.getMBeanServerConnection();
String action = pause ? "pauseConsumption"
: "resumeConsumption";
Exception exception = null;
for (String queueName : queueNames) {
exception = null;
try {
ObjectName destination = findQueueObjectName(con,
jmsModuleName, queueName);
if (pause) {
Object pauseConsResult = con.invoke(destination,
action, null, null);
logger.info("Result from pauseConsumption: "
+ pauseConsResult);
} else {
Object resumeConsResult = con.invoke(destination,
action, null, null);
logger.info("Result from resumeConsumption: "
+ resumeConsResult);
}
updatedQueues.add(new QueueInfo(SYSTEM_OSB, queueName));
} catch (Exception e) {
exception = e;
logger.error("Queue not found name " + jmsModuleName
+ ":" + queueName);
throw e;
} finally {
// Audit.getInst().msg(Audit.AuditAction.PauseQueue,
// action+" of "+jmsModuleName+":"+queueName+" "+(exception==null?"success":"failed"),
// null, exception);
}
}

} catch (Exception ioe) {
// TODO: review
// throw new RuntimeException("Error while pausing the queue",
// ioe);
// ignore the error and continue working with the rest of the
// nodes
logger.error("Error while pausing the queue on " + server, ioe);
} finally {
if (jmxCon != null)
try {
jmxCon.close();
} catch (IOException e) {
logger.error("Error while closing JMX connection", e);
}
}
}
return updatedQueues;
}

private static JMXConnector getJMXConnector(Server server, String username,
String password) throws IOException {
String fullServerURL = "service:jmx:iiop://" + server.getUrl()
+ "/jndi/weblogic.management.mbeanservers.runtime";
JMXConnector jmxCon = null;

JMXServiceURL serviceUrl = new JMXServiceURL(fullServerURL);

Hashtable env = new Hashtable();
env.put(JMXConnectorFactory.PROTOCOL_PROVIDER_PACKAGES,
"weblogic.management.remote");
env.put(javax.naming.Context.SECURITY_PRINCIPAL, username);
env.put(javax.naming.Context.SECURITY_CREDENTIALS, password);

jmxCon = JMXConnectorFactory.newJMXConnector(serviceUrl, env);

return jmxCon;
}

private ObjectName findQueueObjectName(MBeanServerConnection con,
String jmsModuleName, String queueName)
throws MalformedObjectNameException, NullPointerException,
Exception {
ObjectName query_obj_name;
query_obj_name = new ObjectName(
"com.bea:Type=JMSDestinationRuntime,Name=" + jmsModuleName
+ "!*" + queueName + "*,*");
Set objectNames = con.queryNames(query_obj_name, null);
if (objectNames == null || objectNames.size() == 0) {
throw new Exception(
"NotFoundException: Error while getting a reference to the queue (queue "
+ queueName + " not found)");
} else if (objectNames.size() > 1) {
throw new Exception(
"DuplicateFoundException: Error while getting a reference to the queue (more than one queue found named "
+ queueName + ")");
} else {
return objectNames.iterator().next();
}
}

class Server implements Serializable {
private static final long serialVersionUID = -466143988049844989L;

private String name;
private String url;

public Server() {
}

public Server(String name, String url) {
setName(name);
setUrl(url);
}

public void setName(String name) {
this.name = name;
}

public String getName() {
return name;
}

public void setUrl(String url) {
this.url = url;
}

public String getUrl() {
return url;
}

@Override
public String toString() {
return (new StringBuilder()).append(getName()).append(" [")
.append(getUrl()).append("]").toString();
}
}

public class QueueInfo implements Comparable {

private String system;
private String queueName;
private String jndiName;
private Collection serverQueueInfo = new Vector();

public QueueInfo(String system, String queueName) {
this(system, queueName, "");
}

public QueueInfo(String system, String queueName, String jndiName) {
this.system = system;
this.queueName = queueName;
this.jndiName = jndiName;
}

public String getQueueName() {
return queueName;
}

public void setQueueName(String queueName) {
this.queueName = queueName;
}

public Collection getServerQueueInfo() {
return serverQueueInfo;
}

public void setServerQueueInfo(
Collection serverQueueInfo) {
this.serverQueueInfo = serverQueueInfo;
}

public void addSingleServerQueueInfo(String serverName,
SingleServerQueueInfo singleServerQueueInfo) {
serverQueueInfo.add(singleServerQueueInfo);
}

public String getJndiName() {
return jndiName;
}

public void setJndiName(String jndiName) {
this.jndiName = jndiName;
}

@Override
public int compareTo(QueueInfo o) {
return queueName.compareTo(o.getQueueName());
}

@Override
public boolean equals(Object o) {
if (o == null)
return false;
if (o instanceof QueueInfo)
return this.compareTo((QueueInfo) o) == 0;
return false;
}

@Override
public int hashCode() {
return queueName.hashCode();
}

@Override
public String toString() {
return getSystem() + ":" + getQueueName() + ":" + getJndiName();
}

public String getSystem() {
return system;
}

public void setSystem(String system) {
this.system = system;
}
}

public class SingleServerQueueInfo {

private String serverName;
private String queueName;
private String jndiName;
private long messagesCurrent;
private long messagesHigh;
private long messagesPending;
private long consumersCurrent;
private long consumersHigh;
private boolean insertionPaused;
private boolean consumptionPaused;
private boolean productionPaused;

public String getQueueName() {
return queueName;
}

public void setQueueName(String queueName) {
this.queueName = queueName;
}

public long getMessagesCurrent() {
return messagesCurrent;
}

public void setMessagesCurrent(long messagesCurrent) {
this.messagesCurrent = messagesCurrent;
}

public long getMessagesHigh() {
return messagesHigh;
}

public void setMessagesHigh(long messagesHigh) {
this.messagesHigh = messagesHigh;
}

public long getMessagesPending() {
return messagesPending;
}

public void setMessagesPending(long messagesPending) {
this.messagesPending = messagesPending;
}

public long getConsumersCurrent() {
return consumersCurrent;
}

public void setConsumersCurrent(long consumersCurrent) {
this.consumersCurrent = consumersCurrent;
}

public long getConsumersHigh() {
return consumersHigh;
}

public void setConsumersHigh(long consumersHigh) {
this.consumersHigh = consumersHigh;
}

public boolean isInsertionPaused() {
return insertionPaused;
}

public void setInsertionPaused(boolean insertionPaused) {
this.insertionPaused = insertionPaused;
}

public boolean isConsumptionPaused() {
return consumptionPaused;
}

public void setConsumptionPaused(boolean consumptionPaused) {
this.consumptionPaused = consumptionPaused;
}

public boolean isProductionPaused() {
return productionPaused;
}

public void setProductionPaused(boolean productionPaused) {
this.productionPaused = productionPaused;
}

public String getServerName() {
return serverName;
}

public void setServerName(String serverName) {
this.serverName = serverName;
}

public String getJndiName() {
return jndiName;
}

public void setJndiName(String jndiName) {
this.jndiName = jndiName;
}

@Override
public String toString() {
return getServerName() + "::" + getQueueName();
}
}
}

Weblogic JMS and Reporting action

One important fact to consider with the Reporting action is how it interacts with Transactions

10g
Suspend all transactions before continuing

11g
If datasource is XA then continue with transaction
If not. Suspend transaction, and run with local transaction.
Therefore if you always want reporting to work you must use non-XA datasource.

Check out jars in path below to decompile...

Oracle\OSB_10_3_1\osb_10.3\lib\common\reporting-impl.jar\com\bea\wli\reporting\jmsprovider\init\
JmsReportingDataHandler.class

Weblogic Patch levels

To work out the patch level for weblogic
All Weblogic level pathces are output to Admin console log.
However OSB patches are not.

For OSB
Change to the MW_HOME /utils/bsu directory. Enter the bsu.sh command

http://download.oracle.com/docs/cd/E14759_01/doc.32/e14143/quickrefax.htm

Not sure how to run it from command line (possibly repoitn DISPLAY to desktop, and run X server)

Magic Query

To me this query is magic.

We were asked to summarize the number of requests going through our system for a month, but split into daytime (8am – 8pm), and nighttime (8pm – 8am).

Immediately I was thinking about writing a program, but this handy query (Oracle) does it in one go, including backtracking to the last day of the previous month to get values. (i.e. For December, we must get the values from midnight November 30, to 8am Dec1)

SELECT E.interface
, CASE WHEN TO_CHAR(time, 'HH24MISS') < '080000' THEN TO_CHAR(time-1, 'YYYY/MM/DD') ELSE TO_CHAR(time, 'YYYY/MM/DD') END "REPORT_DATE"
, COUNT(CASE WHEN TO_CHAR(time, 'HH24MISS') > '080000' AND TO_CHAR(time, 'HH24MISS') <= '200000' THEN '1' ELSE NULL END) "08AM_08PM"
, COUNT(CASE WHEN TO_CHAR(time, 'HH24MISS') > '200000' OR TO_CHAR(time, 'HH24MISS') <= '080000' THEN '1' ELSE NULL END) "08PM_08AM"
FROM transactions E
,projects P
WHERE E.project_name = P.Project_Name
AND P.seq = 1
AND E.status = 'Entry'
AND E.time > TO_DATE('20101201000000','YYYYMMDDHH24MISS')
AND E.time < TO_DATE('20101231235959','YYYYMMDDHH24MISS')
GROUP BY E.interface
, CASE WHEN TO_CHAR(TIME, 'HH24MISS') < '080000' THEN TO_CHAR(TIME-1, 'YYYY/MM/DD') ELSE TO_CHAR(TIME, 'YYYY/MM/DD') END
ORDER BY 1,2

HTTP GET

Ran into a nasty problem with one of our webapps recently.

We have some jobs that enable and disable a number of weblogic entities (OSB proxies/ JMS queue/ Soa Composites). We noticed that some of these were failing.

The exception we saw was ERROR: Another session operation is in progress. Please retry later.

As part of our process (to disable OSB proxies) we need to create and use a weblogic session. We only create one instance, and then activate the changes at the end. But in our logs a 2^nd session was been created.

After careful inspection it turned out the second session was been generated by a different thread. Checking the access.log confirmed a 2^nd request arriving 30 seconds after the first. (This maybe certain versions of Firefox.. Need more investigation)

This it appears is because we were using a HTTP Get (we should be using Post) for our request. Apparently on seeing no response/ ack etc, the browser was re-submiting the request on its own accord.

So now we had 2 conflicting requests both trying to access the weblogic session. Obviously the 2^nd fails. This was then reported to the user.

Meanwhile the original request succeeds silently.

Our fix is to change the request to a post.

Also adding a token to the post request to prevent against double submit.

….

2011-01-18 20:51:21,335 [DEBUG] myClass:[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)':- Realizing changes in the session: WebAdminTool1295383871917

2011-01-18 20:51:21,759 [DEBUG] myClass:[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)':- Setting service OSBProxy::false to true

2011-01-18 20:51:28,551 [INFO ] ie.bge.middleware.jmswebtool.data.impl.jmx.QueuePersistenceJmxImpl:[ACTIVE] ExecuteThread: '4' for queue: 'weblogic.kernel.Default (self-tuning)':- Result from pauseConsumption: null

2011-01-18 20:51:28,554 [INFO ] ie.bge.middleware.jmswebtool.data.impl.jmx.QueuePersistenceJmxImpl:[ACTIVE] ExecuteThread: '4' for queue: 'weblogic.kernel.Default (self-tuning)':- Result from pauseConsumption: null

...

2011-01-18 20:51:28,644 [DEBUG] myClass :[ACTIVE] ExecuteThread: '4' for queue: 'weblogic.kernel.Default (self-tuning)':- Creating new Weblogic session WebAdminTool1295383888606

2011-01-18 20:51:48,669 [ERROR] myClass:[ACTIVE] ExecuteThread: '4' for queue: 'weblogic.kernel.Default (self-tuning)':- ERROR: Another session operation is in progress. Please retry later.

...

2011-01-18 20:51:58,571 [DEBUG] myClass:[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)':- Activated changes in the session: WebAdminTool1295383871917

Free memory Grid Control Agent

One problem we had with our Linux systems is: “where is all the memory gone?”

We’ve been there in the past and it looks like the Grid Control (GC) Agent suffers from exactly this misunderstanding.

In our case we have:

- Each of our PROD boxes has 32 GB (GiB to be precise) of physical memory

- They all have 16GB of swap space

- None of them is really using the swap space

- Two out of three report less than 1GB of free memory

o one 166MB

o two 720MB

o three 3978MB

- All three are using 20GB or more memory for caching

o one 23 GB

o two 21GB

o three 20GB

So, where is the memory gone? Caching of course!

Check memory using the free command or reading the MemFree line in /proc/meminfo is not good enough w/ Linux systems.

If there is any memory available, the kernel will take it for I/O caching. If another process requests more memory, the kernel will take it out of the chunk used for I/O caching.

Thus, in Linux systems, the memory available to applications is the memory reported to be free PLUS the memory used for I/O caching.

So if we add the two values:

grep ^MemFree /proc/meminfo | awk '{ print $2 }'

grep ^Cached /proc/meminfo | awk '{ print $2 }'

we get the effective memory available to applications (memory free).

Looks like this is the bit where the GC Agents gets confused.

What can we do?

- We could flush the I/O cache and monitor

- We could check with Oracle what the GC Agents is supposed to be reporting (and/or report a bug)

- Just monitor MemFree and Cached and wait for the next GC alert (monitoring should be done by TSG, they already are flooding our system logs w/ SNMP daemon messages)

- A combination of any of the above

HTTP Authentication

Despite working with JEE for years I have always rolled my own Authentication solution. In my current position I inherited a FORM based security solution.

This website give a nice overview.

http://onjava.com/pub/a/onjava/]2002/06/12/form.html

Of particular note:

Auth method is defined in the web.xml in the following section

<login-config>

<auth-method>FORM</auth-method>

<realm-name>myrealm</realm-name>

<form-login-config>

<form-login-page>/login.jsp</form-login-page>

<form-error-page>/fail_login.html</form-error-page>

</form-login-config>

</login-config>

The login form must contain fields for entering username and password.

These fields must be named

j_username and j_password, respectively.

This form should post these values to j_security_check logical name. (Should use ssl to ensure passwords are protected)

In our case we were using this on Welbogic, and using the default myrealm. This then uses the users/ groups and roles as defined in the weblogic domain.

Oracle JCA and OSB

This is a right design cock up on Oracles part if you ask me.

Lets create a new OSB proxy to communicate with a Database table. In past projects we took the custom approach or rolling our own DB adapter, by creating a JAR resource, and accessing it via the OSB proxy.

Architecturally however this approach would have been frowned upon, so in my current position, where I’m inheriting a system built by Oracle consultancy they take the ‘approved’ design approach of using TopLink to create a DB adapter, and communicating with this via JCA.

Unfortunately it is a bit of a disaster.

Roadblock one:

To create/ edit the TopLink adapter we must use JDevelopper.

To create/ edit OSB proxy we must use eclipse or the web front end.

This, oracle owned blog entry even sounds exasperated, but was a life-saver in terms of getting the system up and running.

JMS lifecycle and Searching Filters (Message Selector)

The JMS message lifecycle can be summarized as below with respect to the following two states:

1. A message sent by a JMS producer
* Without any associated transaction:
It is immediately current.
* Within a JTA transaction or transacted JMS session:
It remains pending until the transaction is committed or rolled back. If the transaction is committed, the message becomes available, and if the transaction is rolled back, the message is removed from the destination.
* With a specified TimeToDeliver property:
It remains pending until the TimeToDeliver period expires. This is expected because the purpose of the TimeToDeliver property is to keep the message unavailable to consumers until the specified time period.

2. A message received by a JMS consumer
* Without any associated JTA transaction and in a non-transacted JMS session with an acknowledgement mode of NO_ACKNOWLEDGE or MULTICAST_NO_ACKNOWLEDGE:
It is immediately removed from the destination.
* Within a JTA transaction or transacted JMS session:
It becomes pending until the transaction or session is committed or rolled back. On commit, the message is removed from the destination, and on rollback, the message becomes available again, unless a Redelivery delay is specified, in which case it continues to remain pending until the Redelivery delay.
* Within a non-transacted JMS session with the acknowledgement mode of AUTO_ACKNOWLEDGE, DUPS_OK_ACKNOWLEDGE, or CLIENT_ACKNOWLEDGE:
It becomes pending until the acknowledgement is received.

Message Selector

When searching for messages in JMS queue. There is an expression language for specifying parameters. This is known as Message Selector

This is the format. This is an example from weblogic that I was using.
JMSMessageId = 'ID:<982769.1294675696539.0>'

What confused me was the ID section. I assumed that was the variable name, so I was changing the format to ID=<> etc.

Note can also do logical operations, see http://download.oracle.com/javaee/6/api/javax/jms/Message.html
e.g.
"JMSType = 'car' AND color = 'blue' AND weight > 2500"

State String
The current state of a message, which could be one of DELAYED, EXPIRED, ORDERED, PAUSED, RECEIVE, REDELIVERY_COUNT_EXCEEDED, SEND, TRANSACTION, or VISIBLE.

Note with Weblogic Unit of Order
We were having problems. A standalone utility program is used to display Jms message see this post . It was having problems displaying some messages. One common theme appeared to be that they were in the Ordered state, as opposed to the visible state. This is due to the message using the Unit of Order to ensure ordered delivery.

Another blog post noted (http://forum.springsource.org/showthread.php?t=69398 )that if the message is in the Receive State, then it may be caused by having 2 Weblogic servers with the same name, where one weblogic server is consuming from a queue serverd from another Weblogic server.

Note also the Xml manipulation required on the JMSMessage

private static final Integer JMS_ALL_STATES = new Integer(0x7fffffff);
 
public JmsMessage[] getMEssages(){
try {
 jmxCon = getJMXConnector(server, username, password);
 jmxCon.connect();
 MBeanServerConnection con = jmxCon.getMBeanServerConnection();
 ObjectName destination = findQueueObjectName(con, jmsModuleName, queueName);
 String newCursor = (String) con.invoke(destination, "getMessages", new Object[] { "JMSMessageID='"+msgId+"'", QueuePersistenceJmxImpl.BROWSE_TIMEOUT,  JMS_ALL_STATES}, OP_GET_MESSAGES_ALL_SIGNATURE);
 result.setCursor(newCursor);
 // Should only be one value so no need to sort
 //con.invoke(destination, "sort", new Object[] { newCursor, POSITION_UNDEFINED, DEFAULT_ORDERING_ATTRS, DEFAULT_ORDER }, OP_SORT_SIGNATURE);
 result.setNumMessages((Long) con.invoke(destination, "getCursorSize", new Object[] { newCursor }, OP_GET_CURSOR_SIZE_SIGNATURE));
 Long initialPosition =new Long(0);
 Integer pageSize=new Integer(100);
 data = (CompositeData[]) con.invoke(destination, "getItems", new Object[] { newCursor, initialPosition, pageSize }, OP_GET_ITEMS_SIGNATURE);                                                 
 if (data != null) {
  JmsMessage[] ret = toJmsMessages(data, new CursorParam(destination, con, newCursor));                                                                     
  if(ret!=null){
   if(ret.length>1)
    logger.warn("findMessage (by messageId) returned more than one result"+JavaUtil.toString(ret));
   return ret[0];
  }             
  return null;
 }
} catch (Exception e) {
 // throw new RuntimeException("Error while browsing messages",
 // e);
 // ignore the error and try the next node
 logger.error("Error searching for message " + msgId + " on server " + server, e);
}

private static JMXConnector getJMXConnector(Server server, String username, String password) throws IOException {
 String fullServerURL = "service:jmx:iiop://" + server.getUrl() + "/jndi/weblogic.management.mbeanservers.runtime";
 JMXConnector jmxCon = null;

 JMXServiceURL serviceUrl = new JMXServiceURL(fullServerURL);

 Hashtable env = new Hashtable();
 env.put(JMXConnectorFactory.PROTOCOL_PROVIDER_PACKAGES, "weblogic.management.remote");
 env.put(javax.naming.Context.SECURITY_PRINCIPAL, username);
 env.put(javax.naming.Context.SECURITY_CREDENTIALS, CryptoUtilWrapper.decrypt(password));

 jmxCon = JMXConnectorFactory.newJMXConnector(serviceUrl, env);

 return jmxCon;
}
 
private JmsMessage[] toJmsMessages(CompositeData[] data, CursorParam params) {
 XPathUtils xpathUtils = null;
 Document d;
 JmsMessage[] ret = new JmsMessage[data.length];
 int i=0;
 MBeanServerConnection con = params.getConnection();
 ObjectName destination = params.getDestination();
 try {
  for (CompositeData resItem : data) {
   JmsMessage msg = new JmsMessage();
   d = XMLUtils.stringToDocument((String) resItem.get("MessageXMLText"));
   xpathUtils = new XPathUtils(d, QueuePersistenceJmxImpl.JMS_MESSAGE_NS_CTX);
   xpathUtils.setDocument(d);
   msg.setId(xpathUtils.findValue("/mes:WLJMSMessage/mes:Header/mes:JMSMessageID"));
   msg.setTimestamp(new Date(Long.parseLong(xpathUtils.findValue("/mes:WLJMSMessage/mes:Header/mes:JMSTimestamp"))));
   // Get BOdy
   CompositeData body = (CompositeData) con.invoke(destination, "getMessage", new Object[] { params.getCursor(), msg.getId() }, OP_CURSOR_GET_MESSAGE_SIGNATURE);
   d = XMLUtils.stringToDocument((String)body.get("MessageXMLText"));
   xpathUtils = new XPathUtils(d, QueuePersistenceJmxImpl.JMS_MESSAGE_NS_CTX);
   xpathUtils.setDocument(d);
   msg.setBody(xpathUtils.findValue("/mes:WLJMSMessage/mes:Body/mes:Text"));
   ret[i++]=msg;
  }
  return ret;
 } catch (Exception e) {
     throw new RuntimeException("Error while transforming CompositeData to JmsMessage", e);
 }
}             

Oracle partitioning table

Just a note on how we partitioned a large table (Transaction) in Oracle into monthly partitions/. It involves creating a swap table to store existing data, then placing it into seperate partitions.


Just a note on how we partitioned a large table (Transaction) in Oracle into monthly partitions/.

DROP TABLE TRANSACTION_SWAP
/
ALTER TABLE TRANSACTION DROP CONSTRAINT TRANSACTION_PK
/
DROP INDEX TRANSACTION_IDX1
/
DROP INDEX TRANSACTION_IDX2
/
RENAME TRANSACTION TO TRANSACTION_SWAP
/
REM
REM Now create new version of table, partitioned and with a different primary key
REM

CREATE TABLE TRANSACTION (
"ID" NUMBER NOT NULL ENABLE,
"ORIGINATOR" VARCHAR2(4 BYTE),
"ORIGINATOR_INSTANCE_ID" VARCHAR2(100 BYTE),
"MESSAGE_REF" VARCHAR2(512 BYTE),
"ENTITY_REF" VARCHAR2(512 BYTE),
"TIME" TIMESTAMP(6),
"STATUS" VARCHAR2(5 BYTE),
"JMS_MESSAGE_ID" VARCHAR2(30 BYTE),
"PROJECT_NAME" VARCHAR2(384 BYTE),
"OPERATION" VARCHAR2(150 BYTE),
"INTERFACE" VARCHAR2(20 BYTE))
Partition By Range(Time)
(Partition P_2010_10_31 Values Less Than (To_Timestamp('2010-11-01','YYYY-MM-DD')),
Partition P_2010_11_30 Values Less Than (To_Timestamp('2010-12-01','YYYY-MM-DD')),
Partition P_2010_12_31 Values Less Than (To_Timestamp('2011-01-01','YYYY-MM-DD')),
Partition P_2011_01_31 Values Less Than (To_Timestamp('2011-02-01','YYYY-MM-DD')),
Partition P_High Values Less Than(Maxvalue));
/*
REM
REM Copy over all data before creating indexes - for efficiency reasons
REM Note in some environments, the time can be null on some entries in the table.
REM
REM These rows will be placed in a distinct table
REM

INSERT INTO TRANSACTION
(ID
,ORIGINATOR
,ORIGINATOR_INSTANCE_ID
,MESSAGE_REF
,ENTITY_REF
,TIME
,STATUS
,JMS_MESSAGE_ID
,PROJECT_NAME
,OPERATION
,INTERFACE)
SELECT
ID
,ORIGINATOR
,ORIGINATOR_INSTANCE_ID
,MESSAGE_REF
,ENTITY_REF
,TIME
,STATUS
,JMS_MESSAGE_ID
,PROJECT_NAME
,OPERATION
,INTERFACE
FROM TRANSACTION_SWAP
WHERE TIME IS NOT NULL
/
DROP TABLE TRANSACTION_ARC_NOTIME
/
CREATE TABLE TRANSACTION_ARC_NOTIME
AS
SELECT *
FROM TRANSACTION_SWAP
WHERE TIME IS NULL
/
/*
REM
REM Now create indexes
REM
*/
rem ALTER TABLE TRANSACTION ADD (CONSTRAINT "TRANSACTION_PK" PRIMARY KEY (TIME,ID));
CREATE UNIQUE INDEX TRANSACTION_UID1 ON TRANSACTION(TIME,ID) LOCAL;

ALTER TABLE TRANSACTION ADD (CONSTRAINT TRANSACTION_PK PRIMARY KEY (TIME, ID) USING INDEX TRANSACTION_UID1);

CREATE INDEX TRANSACTION_IDX1 On TRANSACTION (ORIGINATOR_INSTANCE_ID, STATUS) LOCAL;
REM
REM And re-create a grant which is present on production. This may fail elsewhere
REM
GRANT SELECT ON TRANSACTION TO READONLY
/

REM
REM Create the archive table at the same time
REM
CREATE TABLE TRANSACTION_ARC (
"ID" NUMBER NOT NULL ENABLE,
"ORIGINATOR" VARCHAR2(4 BYTE),
"ORIGINATOR_INSTANCE_ID" VARCHAR2(100 BYTE),
"MESSAGE_REF" VARCHAR2(512 BYTE),
"ENTITY_REF" VARCHAR2(512 BYTE),
"TIME" TIMESTAMP(6),
"STATUS" VARCHAR2(5 BYTE),
"JMS_MESSAGE_ID" VARCHAR2(30 BYTE),
"PROJECT_NAME" VARCHAR2(384 BYTE),
"OPERATION" VARCHAR2(150 BYTE),
"INTERFACE" VARCHAR2(20 BYTE))
Partition By Range(Time)
(Partition P_2010_10_31 Values Less Than (To_Timestamp('2010-11-01','YYYY-MM-DD')),
Partition P_2010_11_30 Values Less Than (To_Timestamp('2010-12-01','YYYY-MM-DD')),
Partition P_2010_12_31 Values Less Than (To_Timestamp('2011-01-01','YYYY-MM-DD')),
Partition P_2011_01_31 Values Less Than (To_Timestamp('2011-02-01','YYYY-MM-DD')),
Partition P_High Values Less Than(Maxvalue));

REM
REM recreate any package body with a new version which defaults the time in the table,
REM to make sure there is always a value there
REM
@TrackingProcedureBody

PROMPT *** OPERATION COMPLETE ***
PROMPT
PROMPT *** WHEN HAPPY WITH RESULTS, TRUNCATE THE TABLE TRANSACTION_SWAP

Web Security (and Weblogic)

Web Security (and Weblogic)

When defining security roles for a web app.

1/ In your web.xml
Declare how and where the authentication should occur.

     <login-config>
           <auth-method>FORM</auth-method>
           <realm-name>myrealm</realm-name>
           <form-login-config>
                 <form-login-page>/action/auth/login</form-login-page>
                 <form-error-page>/jsp/security/login-error.jsp</form-error-page>
           </form-login-config>
     </login-config>

2/ In your web.xml
Declare your authorization roles

<security-role>
           <role-name>MessagePoster</role-name>
     </security-role>
     <security-role>
           <role-name>MessageViewer</role-name>
     </security-role>
     <security-role>
           <role-name>QueuePauser</role-name>
     </security-role>
     <security-role>
           <role-name>ServiceEnabler</role-name>
     </security-role>
     <security-role>
           <role-name>LogLevelManipulator</role-name>
     </security-role>
     <security-role>
           <role-name>Resequencer</role-name>
     </security-role>
     <security-role>
           <role-name>MessageRetrier</role-name>
     </security-role>
     <security-role>
           <role-name>ErrorReportCloser</role-name>
     </security-role>
     <security-role>
           <role-name>SystemSwitcher</role-name>
     </security-role>
     <security-role>
           <role-name>OptionsSetter</role-name>
     </security-role>
   

For Weblogic.
In the security realm different security models are possible. This is set from the
Security Model Default: setting
Options are `DD (Deployment Discriptor), Custom Roles, Custom Roles and Policies, Advanced..
When deploying you can set mode
Security
What security model do you want to use with this application?
DD Only: Use only roles and policies that are defined in the deployment descriptors. Custom Roles: Use roles that are defined in the Administration Console; use policies that are defined in the deployment descriptor. Custom Roles and Policies: Use only roles and policies that are defined in the Administration Console. Advanced: Use a custom model that you have configured on the realm's configuration page.

In your Weblogic.xml you can perform the role/ group/ user mappings
(one entry per role)

<wls:security-role-assignment>
     <wls:role-name>ErrorReportCloser</wls:role-name>
     <wls:principal-name>WATSupportGroup</wls:principal-name>
</wls:security-role-assignment>

Or you can delegate the mapping back to the Weblogic admin console (better).. See my earlier post on defining Weblogic Roles
(one entry per role)

<wls:security-role-assignment>
     <wls:role-name>MessageViewer</wls:role-name>
     <externally-defined/>
</wls:security-role-assignment>

Oracke Soa Suite

(work in progress) I will fill this in as I progress

Instructions for Soa Suite

Best place to start is a tutorial

http://download.oracle.com/docs/cd/E12839_01/integration.1111/e10275/intro.htm

This tells you what you need to download

· Task 1: Install Oracle JDeveloper Studio

· Task 2: Install the Fusion Order Demo Application

· Task 3: Install Oracle SOA Suite

· Task 4: Create a Connection to an Oracle WebLogic Server

Task1/

Download from

http://www.oracle.com/technetwork/developer-tools/jdev/overview/index.html

Then you need to download Extensions from (http://www.oracle.com/ocom/groups/public/@otn/documents/webcontent/156082.xml)

Oracle SOA Composite Editor

Pretty print Xml

If you want to format your Xml..theres lots of ways

This url uses Xerces

http://stackoverflow.com/questions/139076/how-to-pretty-print-xml-from-java

(it also uses jtidy which I could not get to work with pure XML.. (kept adding HTML tags).

If you want to avoid xerces and just stick to standard java then you need to use the Transformer.

Just to note that top rated answer requires the use of xerces.

If you don't want to add this dependency then you can simply use the standard jdk libraries.

(Note if an error occurs this will return the original text)

package ie.bge.middleware.tools;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.Source;
import javax.xml.transform.Transformer;
import javax.xml.transform.sax.SAXSource;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.stream.StreamResult;

import org.xml.sax.InputSource;

public class Test {
public static void main(String[] args) {
Test t = new Test();
System.out.println(t.formatXml("text D"));
}

public String formatXml(String xml){
try{
Transformer serializer= SAXTransformerFactory.newInstance().newTransformer();
serializer.setOutputProperty(OutputKeys.INDENT, "yes");
Source xmlSource=new SAXSource(new InputSource(new ByteArrayInputStream(xml.getBytes())));
StreamResult res = new StreamResult(new ByteArrayOutputStream());
serializer.transform(xmlSource, res);
return new String(((ByteArrayOutputStream)res.getOutputStream()).toByteArray());
}catch(Exception e){
//TODO log error
return xml;
}
}

}

Weblogic DB issue

Found on Weblogic 11g

I have a few DataSources defined. Some of which are on the same DB server just different SID’s (Oracle DB’s).

One of the DB instances was removed. Then all instances of DB’s on that server failed to start, (Even though the other DB was still running. When I untargeted the problem server and re-started, then the 2^nd DB started fine.)

Debugging Log4j issues

Log4j

Handy for log4j issues

Set the following switch when you start the JVM to see what log4j is upto.

-Dlog4j.debug

Check for log4j output. This will tell where it is loading its configuration from.

Note be careful with multiple webapps having multiple log4j.xml or log4j.properties. These can override your settings.

This next section is not true.. Its was actually caused by log4j delayed write to log file.. However I'm still including it because the detection techniques are useful.

(My actual problem was incredibly simple, and I shoudl have checked this first, but I made an assumption, and as we all know assumption is the mother of all F£$k ups.. The tool I was working on was using Simple Logging Facade for Java (SLF4J).. The developpers had simply included the slf4j-jdk.jar fil in the classpath instead of the slf4j-log4j.jar. Fixing my problem was simply a case of replacing the wrong jar. USing the log4j debugger was useful however in pinpointing what configuration files were ben picked up.

Weblogic’s many jar files...

Heres some problems I encountered trying to run some applications using Weblogic.

1/

java.lang.NoClassDefFoundError: weblogic/kernel/KernelStatus

I was banging my head against a wall with a

Exception in thread "Main Thread" java.lang.NoClassDefFoundError: weblogic/kernel/KernelStatus

While trying to write a simple standalone program to connect to a Weblogic instance via JNDI, and manipulate some JMX values. The solution was provided via this posting

http://blogs.oracle.com/jamesbayer/2008/08/workshop_for_weblogic_103_jee.html

To actually run the Main method class in Workshop, I right click on the class in the Project Explorer and select Run As->Open Run Dialog. In the classpath tab, be sure to add the wlclient.jar file located here \wlserver_10.3\server\lib to the User Entries section and remove the WebLogic System Libraries from the Bootstrap Entries section. If you forget to remove the WebLogic System Libraries, you will get a stack like this: Exception in thread "Main Thread" java.lang.NoClassDefFoundError: weblogic/kernel/KernelStatus

There is also a wlFullCLient.jar file that may be useful in other scenarios

2/

Just when you thought it was safe however, you get the following

ClassCastException: weblogic.jdbc.common.internal.ConnectionEnv

Here http://forums.oracle.com/forums/thread.jspa?threadID=1048084&tstart=118

the advice is as follows

I would suggest you to remove the wlClient.jar from the classpath.
Also make sure that there are no other jar files related to weblogic in the classpath.
Add weblogic.jar in the classpath and make sure that this weblogic.jar is the same jar that is in the weblogic server classpath.

This happens because of the difference in the jar used by the Client JVM and server JVM.

In fairness this does fix the problem.

Depending on what your client does, it may be necessary to include the Weblogic.jar (if its doing some funky stuff), to ensure it has all the correct classes. Be careful from oracle workshop (eclipse) however of attached bootstrap class loaders.

3/ Missing servlet jars/ JEE standard jars.

You may run into problems with missing J2EE standard jars. I for example was getting errors about HttpServletRequest etc from the Servlet Api. In previous versions of weblogic, the weblogic.jar file would contian all of these, but since 10.3.x they have stopped including it.

When looking up missing jars, try the following, check the following...

$WL_HOME/server/lib/api.jar

In there the MANIFEST.MF contains a list of jars that include the usual J2EE jars etc. These have been removed from Weblogic.jar so may be needed.

Finally... I've mentioned this before but it is important when working with Ears and Wars on weblogic.

If you start seeing exceptions such as

java.lang.ClassCastException: weblogic.xml.jaxp.RegistrySAXParserFactory cannot be cast to javax.xml.parsers.SAXParserFactory

deep in the Weblogic code, you have more than likely run into a dreaded classpath incompatibility.

To avoid this use the following in your weblogic.xml

<wls:container-descriptor>

<wls:prefer-web-inf-classes>true</wls:prefer-web-inf-classes>

</wls:container-descriptor>

This will then result in a lot of ClassNotFoudn exceptions which may mean a hunt through your libs to add missing files

For instance heres the issues I ran into

java.lang.ClassCastException: weblogic.xml.jaxp.RegistrySAXParserFactory cannot be cast to javax.xml.parsers.SAXParserFactory

This means that we need a newer version of the SAXParserFactory than provided by Weblogic (we are using Bea jdk 1.6, so thast is picking up the lastest version of javax.xml.parsers.SAXParserFactory, but then weblogic.xml.jaxp.RegistrySAXParserFactory is been instantiated and this refers to an older version of SAXParserFactory)

The problem in my case was un-resolvable. The incompatible jar was needed to perform translation of a binary Xml format, but unfortunately it also included a newer version of xmlparser. This newer version then started causing problems with taglibs in Jsps. Our fix was to move of deployment from 10.3.1 to 10.3.3

4/ java.lang.NoClassDefFoundError: weblogic/utils/NestedException

After trawling the web there are a lot of references to wlFullClient.jar

This refers to rolling your own version of all wl client jars.

What might be simpler is to simply add the required jars.

This is the list of client jars.

webserviceclient+ssl.jar

webserviceclient.jar

wl-j2ee-client.jar

wlclient.jar

wljmsclient.jar

wljmxclient.jar

wlnmclient.jar

wlsafclient.jar

wlstestclient.ear

wseeclient.jar

For NestedException the required jar is wljmsclient.jar

Tomcat deploy arbitrary dir

To deploy a webapp from a directory not in the appBase (the webapps directory by default) of Tomcat /Catalina, then its simple.

At its most simple you simply create a file in the $CATALINA_BASE/conf/[enginename]/[hostname] directory.

The name of the file is the webapp Context name you want.

The contents must include the docBase attribute. This points the location of the webapp.

This is very useful for development, so every change can be immediately realised on the webapp without restarting, or undeploying/ deploying (which is necessary if you deploy it using Tomcats manager, since it copies the apps into the webapp directory.)

e.g. save SpringSchedule.xml in the $CATALINA_BASE/conf/[enginename]/[hostname]




<Context docBase="F:/eclipse_workspace/SpringSchedule/www">

</Context>

Note: If you add properties files, or update the web.xml you will need to reload the webapp from the tomcat manager console.

Ref

http://tomcat.apache.org/tomcat-7.0-doc/config/context.html