Follow up:

I tried the following, but seemed to always get the default, even on
the mentioned IP:

<VirtualHost 192.168.1.154:443>
# Enable/Disable SSL for this virtual host.
SSLEngine on

SSLCertificateFile /etc/ssl/certs/www.mysite.com.verisign.crt
SSLCertificateKeyFile /etc/ssl/certs/www.mysite.com.key
SSLCertificateChainFile /etc/ssl/certs/verisign.chain.crt

etc
</VirtualHost>

<VirtualHost _default_:443>
SSLCertificateFile /etc/ssl/certs/wildcard.default.com.crt
SSLCertificateKeyFile /etc/ssl/certs/www.mysite.com.key
# The above key is for wildcard.default.com.crt as well
SSLCertificateChainFile /etc/ssl/certs/sf_issuing.crt

etc