button.addActionListener(new ActionListener(){
    public void actionPerformed(ActionEvent e){
        JButton b = (JButton)e.getSource();
        JPanel panel = b.getParent();
        ...
    }
});