Month: October 2013

Sample Java BouncyCastle Algorithm for Encryption / Decryption

Posted on

// Java imports
 import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.NoSuchProviderException;
import java.security.SecureRandom;
import java.security.Security;
import java.util.Iterator;
import sun.misc.BASE64Encoder;
import sun.misc.BASE64Decoder;
// Bouncy castle imports
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openpgp.PGPCompressedData;
import org.bouncycastle.openpgp.PGPCompressedDataGenerator;
import org.bouncycastle.openpgp.PGPEncryptedDataGenerator;
import org.bouncycastle.openpgp.PGPEncryptedDataList;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPLiteralData;
import org.bouncycastle.openpgp.PGPObjectFactory;
import org.bouncycastle.openpgp.PGPOnePassSignatureList;
import org.bouncycastle.openpgp.PGPPrivateKey;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.bouncycastle.openpgp.PGPPublicKeyEncryptedData;
import org.bouncycastle.openpgp.PGPPublicKeyRing;
import org.bouncycastle.openpgp.PGPPublicKeyRingCollection;
import org.bouncycastle.openpgp.PGPSecretKey;
import org.bouncycastle.openpgp.PGPSecretKeyRingCollection;
import org.bouncycastle.openpgp.PGPUtil;
//
public class SingleSignOnTest {
      private static File publicKeyFile = new File("/Development/Java/Single Sign On with Encryption(PGP)/PGP1D0.pkr");
      private static File privateKeyFile = new File("/Development/Java/Single Sign On with Encryption(PGP)/PGP1D0.skr");
      private static String privateKeyPassword = "passphrase";
 
       //
      // Public class method decrypt
      //
      public static String decrypt(byte[] encdata) {
              System.out.println("decrypt(): data length=" + encdata.length);
              // ----- Decrypt the file
              try {
                      ByteArrayInputStream bais = new ByteArrayInputStream(encdata);
                      FileInputStream privKey = new FileInputStream(privateKeyFile);
                      return _decrypt(bais, privKey, privateKeyPassword.toCharArray());
              } catch (Exception e) {
                      System.out.println(e.getMessage());
                      e.printStackTrace();
              }
              return null;
      }
      //
      // Public class method encrypt
      //
      public static byte[] encrypt(byte[] data) {
              try
              {
                      // ----- Read in the public key
                      PGPPublicKey key = readPublicKeyFromCol(new FileInputStream(publicKeyFile));
                      System.out.println("Creating a temp file...");
                      // create a file and write the string to it
                      File tempfile = File.createTempFile("pgp", null);
                      FileOutputStream fos = new FileOutputStream(tempfile);
                      fos.write(data);
                      fos.close();
                      System.out.println("Temp file created at ");
                      System.out.println(tempfile.getAbsolutePath());
                      System.out.println("Reading the temp file to make sure that the bits were written\n--------------");
                      BufferedReader isr = new BufferedReader(new FileReader(tempfile));
                      String line = "";
                      while ( (line = isr.readLine()) != null )
                      {
                              System.out.println(line + "\n");
                      }
                      // find out a little about the keys in the public key ring
                      System.out.println("Key Strength = " + key.getBitStrength());
                      System.out.println("Algorithm = " + key.getAlgorithm());
                      System.out.println("Bit strength = " + key.getBitStrength());
                      System.out.println("Version = " + key.getVersion());
                      System.out.println("Encryption key = " + key.isEncryptionKey()+ ", Master key = " + key.isMasterKey());
                      int count = 0;
                      for ( java.util.Iterator iterator = key.getUserIDs(); iterator.hasNext(); )
                      {
                              count++;
                              System.out.println((String) iterator.next());
                      }
                      System.out.println("Key Count = " + count);
                      // create an armored ascii file
                      // FileOutputStream out = new FileOutputStream(outputfile);
                      // encrypt the file
                      // encryptFile(tempfile.getAbsolutePath(), out, key);
                      // Encrypt the data
                      ByteArrayOutputStream baos = new ByteArrayOutputStream();
                      _encrypt(tempfile.getAbsolutePath(), baos, key);
                      System.out.println("encrypted text length=" + baos.size());			
                      tempfile.delete();
                      return baos.toByteArray();
              }
              catch (PGPException e)
              {
                      // System.out.println(e.toString());
                      System.out.println(e.getUnderlyingException().toString());
                      e.printStackTrace();
              }
              catch (Exception e)
              {
                      e.printStackTrace();
              }
              return null;
      }
      //
      // Private class method readPublicKeyFromCol
      //
      private static PGPPublicKey readPublicKeyFromCol(InputStream in)
                     throws Exception {
              PGPPublicKeyRing pkRing = null;
              PGPPublicKeyRingCollection pkCol = new PGPPublicKeyRingCollection(in);
              System.out.println("key ring size=" + pkCol.size());
              Iterator it = pkCol.getKeyRings();
              while (it.hasNext()) {
                      pkRing = (PGPPublicKeyRing) it.next();
                      Iterator pkIt = pkRing.getPublicKeys();
                      while (pkIt.hasNext()) {
                              PGPPublicKey key = (PGPPublicKey) pkIt.next();
                              System.out.println("Encryption key = " + key.isEncryptionKey() + ", Master key = " + 
                                                 key.isMasterKey());
                              if (key.isEncryptionKey())
                                      return key;
                      }
              }
              return null;
      }
      //
      // Private class method _encrypt
      //
      private static void _encrypt(String fileName, OutputStream out, PGPPublicKey encKey)
                          throws IOException, NoSuchProviderException, PGPException
      {
              out = new DataOutputStream(out);
              ByteArrayOutputStream bOut = new ByteArrayOutputStream();
              System.out.println("creating comData...");
              // get the data from the original file
              PGPCompressedDataGenerator comData = new PGPCompressedDataGenerator(PGPCompressedDataGenerator.ZIP);
              PGPUtil.writeFileToLiteralData(comData.open(bOut), PGPLiteralData.BINARY, new File(fileName));
              comData.close();
              System.out.println("comData created...");
              System.out.println("using PGPEncryptedDataGenerator...");
              // object that encrypts the data
              PGPEncryptedDataGenerator cPk = new PGPEncryptedDataGenerator(PGPEncryptedDataGenerator.CAST5, 
                                              new SecureRandom(), "BC");
              cPk.addMethod(encKey);
              System.out.println("used PGPEncryptedDataGenerator...");
              // take the outputstream of the original file and turn it into a byte
              // array
              byte[] bytes = bOut.toByteArray();
              System.out.println("wrote bOut to byte array...");
              // write the plain text bytes to the armored outputstream
              OutputStream cOut = cPk.open(out, bytes.length);
              cOut.write(bytes);
              cPk.close();
              out.close();
      }
      //
      // Private class method _decrypt
      //
      private static String _decrypt(InputStream in, InputStream keyIn,
                      char[] passwd) throws Exception {
              in = PGPUtil.getDecoderStream(in);
              try {
                      PGPObjectFactory pgpF = new PGPObjectFactory(in);
                      PGPEncryptedDataList enc;
                      Object o = pgpF.nextObject();
                      //
                      // the first object might be a PGP marker packet.
                      //
                      if (o instanceof PGPEncryptedDataList) {
                              enc = (PGPEncryptedDataList) o;
                      } else {
                              enc = (PGPEncryptedDataList) pgpF.nextObject();
                      }
                      //
                      // find the secret key
                      //
                      Iterator it = enc.getEncryptedDataObjects();
                      PGPPrivateKey sKey = null;
                      PGPPublicKeyEncryptedData pbe = null;
                      while (sKey == null && it.hasNext()) {
                              pbe = (PGPPublicKeyEncryptedData) it.next();
                              System.out.println("pbe id=" + pbe.getKeyID());
                              sKey = findSecretKey(keyIn, pbe.getKeyID(), passwd);
                      }
                      if (sKey == null) {
                             throw new IllegalArgumentException("secret key for message not found.");
                      }
                      InputStream clear = pbe.getDataStream(sKey, "BC");
                      PGPObjectFactory plainFact = new PGPObjectFactory(clear);
                      Object message = plainFact.nextObject();
                      if (message instanceof PGPCompressedData) {
                              PGPCompressedData cData = (PGPCompressedData) message;
                              PGPObjectFactory pgpFact = new PGPObjectFactory(cData.getDataStream());
                              message = pgpFact.nextObject();
                      }
                      ByteArrayOutputStream baos = new ByteArrayOutputStream();
                      if (message instanceof PGPLiteralData) {
                              PGPLiteralData ld = (PGPLiteralData) message;
                              InputStream unc = ld.getInputStream();
                              int ch;
                              while ((ch = unc.read()) >= 0) {
                                      baos.write(ch);
                              }
                      } else if (message instanceof PGPOnePassSignatureList) {
                              throw new PGPException("encrypted message contains a signed message - not literal data.");
                      } else {
                              throw new PGPException("message is not a simple encrypted file - type unknown.");
                      }
                      if (pbe.isIntegrityProtected()) {
                              if (!pbe.verify()) {
                                      System.err.println("message failed integrity check");
                              } else {
                                      System.err.println("message integrity check passed");
                              }
                      } else {
                              System.err.println("no message integrity check");
                      }
                      return baos.toString();
              } catch (PGPException e) {
                      System.err.println(e);
                      if (e.getUnderlyingException() != null) {
                              e.getUnderlyingException().printStackTrace();
                      }
              }
              return null;
      }
      //
      // Private class method findSecretKey
      //
      private static PGPPrivateKey findSecretKey(InputStream keyIn, long keyID,
                      char[] pass) throws IOException, PGPException,
                      NoSuchProviderException {
              PGPSecretKeyRingCollection pgpSec = new PGPSecretKeyRingCollection(PGPUtil.getDecoderStream(keyIn));
              PGPSecretKey pgpSecKey = pgpSec.getSecretKey(keyID);
              if (pgpSecKey == null) {
                      return null;
              }
              return pgpSecKey.extractPrivateKey(pass, "BC");
      }
      //
      // Public class method readFile
      //
      public byte[] readFile(File file) throws IOException {
              FileInputStream fis = new FileInputStream(file);
              byte[] buf = new byte[4096];
              int numRead = 0;
              ByteArrayOutputStream baos = new ByteArrayOutputStream();
              while ((numRead = fis.read(buf)) > 0) {
                      baos.write(buf, 0, numRead);
              }
              fis.close();
              byte[] returnVal = baos.toByteArray();
              baos.close();
              return returnVal;
      }
      //
      // Public main method
      //
      public static void main(String[] args) {
              Security.addProvider(new BouncyCastleProvider());
              String TOKEN = "aamine";
              // ----- Encrypt the message to a file
              // them
              byte[] encdata = encrypt(TOKEN.getBytes());
              System.out.println("Encrypted: " + encdata);
              // Encode the byte array to a string
              BASE64Encoder en = new BASE64Encoder();		
              String temp = en.encode(encdata);
              System.out.println("Temp: " + temp);
              // us
              byte[] newB=null;		
              BASE64Decoder en1 = new BASE64Decoder();
              try {
                      newB = en1.decodeBuffer(temp);
              } catch (Exception e) {
                      System.out.println("Exception: " + e);
              }
              System.out.println("byte array" + newB.length);
              // ----- Decrypt the token that
              String result = decrypt(newB);
              System.out.println("Decrypted: " + result);
      }
}

7 Tips for Exception Handling in GWT | Summa Blog – Mozilla Firefox

Posted on Updated on

7 Tips for Exception Handling in GWT | Summa Blog – Mozilla Firefox.

Exception handling best practices in Java are pretty straight-forward by now, and for the most part this carries over into the world of GWT (since GWT is, from the end developer’s perspective, just Java!). There are, however, a few tricks and gotchas that are important to be aware of. In this blog post I’ll describe a few that I’ve encountered, but please feel free to share your own.

Starting with the client tier and working my way to the server, here we go…

1. Handle all uncaught client-side exceptions

The first thing and simplest thing to do is to ensure that all unhandled exceptions bubbling out of the client tier are caught. You can accomplish this by registering an UncaughtExceptionHandler, like this:

  1. public class Example implements EntryPoint {  
  2.   public void onModuleLoad() {  
  3.     GWT.setUncaughtExceptionHandler(new   
  4.       GWT.UncaughtExceptionHandler() {  
  5.       public void onUncaughtException(Throwable e) {  
  6.         // do exception handling stuf  
  7.     }  
  8.     // do module loading stuff  
  9.   }    

Pretty straight-forward. What happens if you don’t set the UncaughtExceptionHandler in your application? Well, in hosted mode, the default handler will conveniently log the exception and stack trace to the console. In web mode, however, the handler is null, and so exceptions “escape” to your browser, which is likely not the user experience you’re looking for (though it does provide the opportunity to use a Javascript debugger for further diagnosis).

Also, be aware that your custom UncaughtExceptionHandler only takes effect the onModuleLoad() returns. In other words, if there was an exception raised where the comment “do module loading stuff” is in the example above, this exception would be handled using your custom UncaughtExceptionHandler that you just defined. This is a known issue, and will not be resolved. As a work-around, if you expect exceptions could be thrown within onModuleLoads(), then you can extract the guts of onModuleLoad() to another method, and invoke it using a DeferredCommand:

  1. public class Example implements EntryPoint {  
  2.   public void onModuleLoad() {  
  3.     GWT.setUncaughtExceptionHandler(new   
  4.       GWT.UncaughtExceptionHandler() {  
  5.       public void onUncaughtException(Throwable e) {  
  6.         // do exception handling stuff  
  7.       }  
  8.     });  
  9.     DeferredCommand.addCommand(new Command() {  
  10.       public void execute() {  
  11.         onModuleLoad2();  
  12.       }  
  13.     });  
  14.   }  
  15.   
  16.   private void onModuleLoad2() {  
  17.     // do module loading stuff  
  18.   }  
  19. }  

2. Unravel GWT’s UmbrellaException

Once your UncaughtExceptionHandler is set, you may notice that some of the exceptions caught are instances of GWT’s UmbrellaException, rather than the actual exception thrown within your code. GWT uses UmbrellaExceptions to wrap exceptions in certain cases (loops, etc.), and it nests the root exception within it. For the purpose of easier debugging and more accurate error handling, it’s often helpful to unwrap these exceptions, like this:

  1. GWT.setUncaughtExceptionHandler(new   
  2.   GWT.UncaughtExceptionHandler() {  
  3.   public void onUncaughtException(Throwable e) {  
  4.     Throwable unwrapped = unwrap(e);  
  5.     // do exception handling stuff  
  6.   }    
  7.     
  8.   public Throwable unwrap(Throwable e) {   
  9.     if(e instanceof UmbrellaException) {   
  10.       UmbrellaException ue = (UmbrellaException) e;  
  11.       if(ue.getCauses().size() == 1) {   
  12.         return unwrap(ue.getCauses().iterator().next());  
  13.       }  
  14.     }  
  15.     return e;  
  16.   }  
  17. }  

3. Log exceptions on the client

Now that you have a handler catching your rogue client side exceptions and also a handy method to unravel them to their root causes, you’ll want to actually useful with these exceptions to help developers more easily diagnose and resolve the underlying problems. While the appropriate action may differ per application (e.g. present an error dialog, email the support team, etc.), at the very least the exception should be logged. Fortunately, GWT delivers a simple but powerful solution for client-side logging, piggy-backing off of the standard java.util.logging package. Plugging a logger into the UncaughtExceptionHandler is easily accomplished:

  1. public class Example implements EntryPoint {  
  2.   
  3.   private Logger logger = Logger.getLogger(“”);  
  4.   
  5.   public void onModuleLoad() {  
  6.    GWT.setUncaughtExceptionHandler(new   
  7.      GWT.UncaughtExceptionHandler() {  
  8.      public void onUncaughtException(Throwable e) {  
  9.         logger.log(Level.ERROR, “Ex caught!”, e);  
  10.     }  
  11.     // do module loading stuff  
  12.   }    
  13. }  

Leveraging a helpful set of handlers that GWT provides, loggers can write to a number of different channels: development mode window, console, in-window pop-up, Firebug, and, the coup-de-gras, to the server(which I’ll cover in a bit). These channels are easily configurable via your .gwt.xml file:

  1. <inherits name=”com.google.gwt.logging.Logging” />  
  2. <set-property name=”gwt.logging.logLevel”   
  3.   value=”INFO” />            
  4. <set-property name=”gwt.logging.enabled”   
  5.   value=”TRUE” />   
  6. <set-property name=”gwt.logging.developmentModeHandler”   
  7.   value=”ENABLED” />    
  8. <set-property name=”gwt.logging.systemHandler”   
  9.   value=”DISABLED” />  
  10. <set-property name=”gwt.logging.popupHandler”   
  11.   value=”DISABLED” />  
  12. <set-property name=”gwt.logging.consoleHandler”   
  13.   value=”ENABLED”/>   
  14. <set-property name=”gwt.logging.firebugHandler”   
  15.   value=”ENABLED” />  

Logging in GWT is very well documented, so there’s no need to retread here.

4. Log exceptions on the server

The majority of the logging handlers write to the client side, however in a production environment this often isn’t much help to you. When your users stumble upon an error, it’s unlikely (and actually unsafe from a security perspective!) to expect them to look in client-side log files to help you diagnose the problem. Instead, your application should do this work for you, by logging all unexpected exceptions to the server. Fortunately, remote logging of this sort is as simple as enabling it in the .gwt.xml config file:

  1. <set-property name=”gwt.logging.simpleRemoteHandler”   
  2.     value=”ENABLED” />    

…and then defining the remote logging servlet in your web.xml file:

  1. <servlet>  
  2.   <servlet-name>remoteLogging</servlet-name>  
  3.   <servlet-class>com.google.gwt.logging.server.RemoteLoggingServiceImpl</servlet-class>  
  4. </servlet>  
  5. <servlet-mapping>  
  6.   <servlet-name>remoteLogging</servlet-name>  
  7.   <url-pattern>/your-gwt-module-name/remote_logging</url-pattern>  
  8. </servlet-mapping>  

Now, with one simple call to the log() method in your UncaughtExceptionHandler, the exception is sent automatically to your server log. Note, however, that by default it’s assumed that you’re using java.util.logging, which may not necessarily be the case. To hook these logs into, say, Log4J, check out SL4J.

Finally, it’s possible that instead of using GWT’s remote logging, you may want to implement a simple custom RPC to log (and possibly do other things – e.g. send email, etc.) when an unexpected exception is encountered. For example:

  1. GWT.setUncaughtExceptionHandler(new   
  2.   GWT.UncaughtExceptionHandler() {  
  3.   public void onUncaughtException(Throwable e) {  
  4.     myService.handleException(e);  
  5.   }  
  6. }  

Seems straight-forward, right? Be careful! First, not all exceptions are serializable or follow the rules of serializable (ugh!). For example, GWT’s JavascriptException inherits Serializable, but doesn’t implement a public no-arg constructor, so when you try to send this over the wire via your RPC, a run-time Serialization exception is thrown. And since this exception would be thrown inside your exception handler, it would effectively swallowed. Second, the stack trace within an exception is transient, and so is lost from client to server (so if you need it on the server side, send it as a separate parameter). Which leads me to the next tip…

5. Consider logging exception stack traces

Of course knowing that an exception occurred isn’t usually helpful unless you know exactly where it was thrown from! On the server side we take this for granted, since stack traces are typically logged by default with our exceptions. On the client side with GWT side, however, it’s not so easy. When Java code gets compiled into Javascript, it is both shrunk and obfuscated. This is obviously useful for performance and security, but from the perspective of supporting applications, it makes things much trickier. For example, instead of seeing a stack trace with method/class names and line numbers, you might get something like this:

 Unknown.Jq(Unknown Source) 
 Unknown.Tm(Unknown Source) 
 Unknown.Sm(Unknown Source)

Not too helpful! Fortunately, there are different options for how GWT compiles into Javascript which provide different amounts of information for debugging. Of course nothing in life is free; the trade-offs are security and performance. If you wanted nice clean Javascript that resembles your Java code and that produces nice clean stack traces, then you can flip the GWT compiler “style” flag to either “pretty” or “detailed”, however this would obviously increase the size the Javascript (and thus take longer to download)…and any user could more easily grock your client side code. There’s a nice compromise though, which is to provide file names and/or line numbers in your stack traces, but to keep the source code obfuscated. This can be configured in your .gwt.xml file:

  1. <set-property name=”compiler.stackMode”   
  2.      value=”emulated” />  
  3. <set-configuration-property name=”compiler.emulatedStack.recordLineNumbers”   
  4.      value=”true”/>   
  5. <set-configuration-property name=”compiler.emulatedStack.recordFileNames”   
  6.      value=”true”/>   

Note, again, that this does increase the size of your Javascript bundle. For one application I worked on, I found that just turning on the emulatedStack increased the Javascript by 30%, and turning on emulatedStack plus line numbers increased it by 100% (i.e. doubled the size!). Depending on the performance requirements of your application, this may or may not be acceptable.

Finally, it’s helpful to know that even with the obfuscated stack traces, there is still hope for hunting down mysterious exceptions. What seems like jibberish is just obfuscated code, and you can manually translate the obfuscated method names (e.g. “Jq”, “Tm”, or “Sm” from the above example) to the Java method names using the symbolMaps file that GWT generates. If only there were a way to do this programmatically…

6. Use the StackTraceDeobfuscator

GWT provides a utility class called the StackTraceDeobfuscator which (as it advertises!) deobfuscates stack traces, translating the condensed, jibberish stack trace (like above) into a one that you would expect. If you’re using GWT’s remote logger, then it helpfully uses this by default, with one caveat: it knows where to find the symbolMaps file. GWT places this in the “WEB-INF/deploy/your-app/symbolMaps” directory, and you can instruct the RemoteLoggerService to look there accordingly:

  1. <init-param>  
  2.   <param-name>symbolMaps</param-name>  
  3.   <param-value>your-app/symbolMaps</param-value</param-value>  
  4. </init-param>  

You could also call the StackTraceDeobfuscator manually, if, for example, you had your own RPC service that handled exceptions in some custom way. I wrote a simple subclass of the StackTraceDeobfuscator, and called it like this:

  1. public class LogServiceImpl extends LogService {   
  2.   public void logError(Throwable t, String stackTrace) {  
  3.     StackTraceElement[] sts = stackTraceDeobfuscator.deobfuscateStackTrace(t.getStackTrace());  
  4.     t.setStackTrace(sts);  
  5.   
  6.     LOG.logError(“An error happened”, t);  
  7.   }  
  8. }  

7. Throw exceptions explicitly in RPC Services

Now, to the server side (which, thankfully, is much less tricky!). Imagine you have a simple RPC service that throws a custom exception, MyRuntimeException, which extends from RuntimeException. For example:

  1. public class MyServiceImpl extends MyService {   
  2.   public void foo() {   
  3.     throw new MyRuntimeException();  
  4.   }  
  5. }  

By Java’s rules, there is no obligation that you explicitly declare (either in the impl or interface) that foo throws MyRuntimeException. For the sake of simplicity, you may choose not to, however you might be surprised then to find that when you catch this exception in your AsyncCallback

  1. myService.foo(new AsyncCallback<Void>() {   
  2.   public void onSuccess(void v) {  
  3.   }  
  4.   
  5.   public void onFailure(Throwable t) {  
  6.     if(t instanceof MyRuntimeException) {   
  7.       // do something  
  8.     }  
  9.   }  
  10. }  

…the object t will not be an instance of MyRuntimeException, but rather an UnexpectedException. This is because GWT did not know to compile MyRuntimeException into Javascript, and so the client code does not know of this type. To remedy, make sure you explicitly throw MyRuntimeException in your service interface (but not in your Async interface!):

  1. public interface MyService {   
  2.   void foo() throws MyRuntimeException;  
  3. }  

Note that if you threw some subclass of MyRuntimeException, the instanceof check work, however be aware that GWT would then compile all concrete subclasses of MyRuntimeException, which could slow build time.

Whew…that’s it. Please share any tips you have, or let me know if you see anything awry with what I’ve written. Thanks!

 

geospatial – What SQL datatype should be used to populate a Solr location (spatial) field when using a DataImportHandler? – Stack Overflow – Mozilla Firefox

Link Posted on Updated on

geospatial – What SQL datatype should be used to populate a Solr location (spatial) field when using a DataImportHandler? – Stack Overflow – Mozilla Firefox

I have a Solr schema which contains a location field (using the default solr.LatLonType ):

<field name="latlng" type="location" indexed="true" stored="true"/>

And I am trying to populate it using a DataImportHandler. Currently I SELECT the value as nvarchar in the format of 17.74628,-64.70725; however it is not populating the Solr field (it remains empty).

What type and format should this column be in to update the location field in Solr?

Answer:

solr.LatLonType is a multi-dimensional type; You can define the field type as:

<fieldType name="location" subFieldSuffix="_coordinate"/>

Using your field name of “latlng” the schema for the coordinate fields will look like this (notice the “subFieldSuffix” used for the 2 dimensional field type solr.LatLonType):

<field name="latlng" type="location" indexed="true" stored="true" />
<field name="latlng_0_coordinate" type="double" indexed="true" stored="true" />
<field name="latlng_1_coordinate" type="double" indexed="true" stored="true" />

“latlng_0_coordinate” should be the latitude and “latlng_1_coordinate” should be the longitude. Your select statement should load “latlng_0_coordinate” and “latlng_1_coordinate” as doubles.

the previous answer works since you’re manually creating the fields that Solr uses to store the lat and long individually, however there’s a dynamic field for that purpose.

<!-- Type used to index the lat and lon components for the "location" FieldType --> <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false" />

If you check field type location, you might find it uses the suffix _coordinate for their values:

<!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->

<fieldType name="location" subFieldSuffix="_coordinate"/>

that works for me in Solr 4 beta, and I believe is present since Solr 3.6 or even older. Anyway, just another solution!

[solr] Indexed and Stored field configuration

Posted on

In a nutshell – an indexed field is searchable, and a stored field has its content stored in the index so it is retrievable. Here are some examples that will hopefully give you a feel for how to set the indexed and stored options: 

indexed=”true” stored=”true” 
Use this for information you want to search on and also display in search results – for example, book title or author. 

indexed=”false” stored=”true” 
Use this for fields that you want displayed with search results but that don’t need to be searchable – for example, destination URL, file system path, time stamp, or icon image. 

indexed=”true” stored=”false” 
Use this for fields you want to search on but don’t need to get their values in search results. Here are some of the common reasons you would want this: 

Large fields and a database: Storing a field makes your index larger, so set stored to false when possible, especially for big fields. For this case a database is often used, as the previous responder said. Use a separate identifier field to get the field’s content from the database. 

Ordering results: Say you define field name=”bookName” type=”text” indexed=”true” stored=”true” that is tokenized and used for searching. If you want to sort results based on book name, you could copy the field into a separate nonretrievable, nontokenized field that can be used just for sorting – 
field name=”bookSort” type=”string” indexed=”true” stored=”false” 
copyField source=”bookName” dest=”bookSort” 

Easier searching: If you define the field <field name=”text” type=”text” indexed=”true” stored=”false” multiValued=”true”/> you can use it as a catch-all field that contains all of the other text fields. Since solr looks in a default field when given a text query without field names, you can support this type of general phrase query by making the catch-all the default field. 

indexed=”false” stored=”false” 
Use this when you want to ignore fields. For example, the following will ignore unknown fields that don’t match a defined field rather than throwing an error by default. 
fieldtype name=”ignored” stored=”false” indexed=”false” 
dynamicField name=”*” type=”ignored” 



Here is a summary of available options on a field, broken down by use case. A true or false indicates that the option must be set to the given value for the use case to function correctly.
use case
indexed
stored
multiValued
omitNorms
termVectors
termPositions
termOffsets

search within field
true

retrieve contents
true

use as unique key
true
false

sort on field
true
false
true [1]

use field boosts
false

document boosts affect searches within field
false

highlighting
true[3]
true
[2]
[2]
[2]

faceting
true

add multiple values, maintaining order
true

field length affects doc score
false

true[5]
true[5]

term frequency[4]
true

document frequency[4]
true

tf*idf[4]
true

term postitions[4]
true
true
true

term offsets[4]
true
true
true
Notes:
  1. recommended but not necessary
  2. stored must always be true for highlighting. If you also add both termVectors and termOffsets, this can be used to boost performance. (Without termVectors/termOffsets, Solr needs to reanalyze the whole field to perform highlighting.) If you furthermore add termPositions, additional speedup may be possible. Note, that you must index the field in order to be able to use termVectors, termOffsets and termPositions.
  3. a tokenizer must be defined for the field, but it doesn’t need to be indexed
  4. For use with the TermVectorComponent
  5. Uses the term vector if present, otherwise the stored field. Reanalyzes the document if using the stored field.

For further considerations for faceting, see also SolrFacetingOverview. For more information on term frequency, positions, offsets etc. see TermVectorComponent.

[mysql] Show all column from all tables

Posted on

Sometime when we work on a large database structure that contains many tables, we don’t know the relationship between tables. If the architect never write any documentation, we may have difficulties to trace the relationship.

Luckily, most of the Database system allow us to query the columns and tables name in the database.
Following is the SQL script to select all tables from a MySQL database.

select * from information_schema.columns where table_schema = ‘l2gs’
 order by table_name,ordinal_position

So when we are frustrated on how  a table is related to another table, we can easily filter by the column name, hopefully the database architect create the same column name for the related table

select * from information_schema.columns where table_schema = ‘l2gs’and column_name like ‘%REASON%’ order by table_name,ordinal_position




Hope it helps all the developers who have same problem